docker+k8s+flannel+ansible

all: 
children: 
etcd: 
hosts: 
192.168.144.128: 
ETCD_NODE_NAME: etcd1 
192.168.144.129: 
ETCD_NODE_NAME: etcd2 
k8s-master: 
hosts: 
192.168.144.128: 
API_NODE_NAME: k8s-master1 
192.168.144.129: 
API_NODE_NAME: k8s-master2 

[root@k8s-master1 tasks]# cat main.yml 
- name: print TMP_NODE value 
debug: 
msg: "{{TMP_NODES}}" 
- name: prepare some dirs 
file: name={{ item }} state=directory 
with_items: 
- "/var/lib/etcd" # etcd ¹¤×÷Ŀ¼ 
- name: download etcd 
copy: src=../files/{{ item }} dest=/usr/bin/{{ item }} mode=0755 
with_items: 
- etcd 
- etcdctl 
- name: create systemd unit 
template: src=etcd.service.j2 dest=/etc/systemd/system/etcd.service 
- name: enable etcd boot 
shell: systemctl enable etcd 
ignore_errors: true 
- name: restart etcd 
shell: systemctl daemon-reload && systemctl restart etcd 
ignore_errors: true 
- name: wait etcd running 
shell: "systemctl status etcd.service|grep Active" 
register: etcd_status 
until: '"running" in etcd_status.stdout' 
retries: 8 
delay: 8 
[root@k8s-master1 tasks]# etcdctl member list 
27d9777c62a65711: name=etcd1 peerURLs=http://192.168.144.128:2380 clientURLs=http://192.168.144.128:2379 isLeader=false 
a18ccad74b84a0ab: name=etcd2 peerURLs=http://192.168.144.129:2380 clientURLs=http://192.168.144.129:2379 isLeader=true 
[root@k8s-master1 tasks]# etcdctl cluster-health 
member 27d9777c62a65711 is healthy: got healthy result from http://192.168.144.128:2379 
member a18ccad74b84a0ab is healthy: got healthy result from http://192.168.144.129:2379 
cluster is healthy 

--- 
- include_tasks: setup-RedHat.yml 
when: ansible_os_family == 'RedHat' 
- name: Install Docker. 
package: 
name: "{{ docker_package }}" 
state: "{{ docker_package_state }}" 
notify: restart docker 
- name: Ensure Docker is started and enabled at boot. 
service: 
name: docker 
state: "{{ docker_service_state }}" 
enabled: "{{ docker_service_enabled }}" 
- name: Ensure handlers are notified now to avoid firewall conflicts. 
meta: flush_handlers 

[root@k8s-master1 tasks]# cat main.yml 
--- 
- include: openssl.yml 
- include: setup.yml 
[root@k8s-master1 tasks]# cat openssl.yml 
- name: print node name11 
debug: 
msg: "{{ API_NODE_NAME }}" 
- name: create ca dir 
file: name={{ item }} state=directory 
with_items: 
- /root/ssl 
- name: ·Ö·¢caÖ¤Êé 
copy: src=../files/{{ item }} dest=/root/ssl/{{ item }} mode=0755 
with_items: 
- ca.key 
- ca.crt 
- name: print node name 
debug: 
msg: "{{ API_NODE_NAME }}" 
- name: ´´½¨ kubernetes Ö¤ÊéÇ©ÃûÇëÇó 
template: src=apiserver_ssl.cnf.j2 dest=/root/ssl/apiserver_ssl.cnf 
- name: ´´½¨Ë½Ô¿ 
shell: 
chdir=/root/ssl 
openssl genrsa -out apiserver.key 2048 
- name: ´´½¨ÇëÇóÖ¤Êé 
shell: 
chdir=/root/ssl 
openssl req -new -key apiserver.key -subj "/CN={{ API_NODE_NAME }}" -config apiserver_ssl.cnf -out apiserver.csr 
- name: ´´½¨Ö¤Êé 
shell: 
chdir=/root/ssl 
openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile apiserver_ssl.cnf -out apiserver.crt 

[root@k8s-master1 tasks]# cat main.yml 
- name: ÏÂÔØ kubelet,kube-proxy ¶þ½øÖƺͻù´¡ cni plugins 
copy: src=../files/{{ item }} dest=/usr/bin/{{ item }} mode=0755 
with_items: 
- kubectl 
- kubelet 
- kube-proxy 
- name: ·Ö·¢ kubeconfigÅäÖÃÎļþ 
template: src=kubeconfig.j2 dest=/root/kubernetes/kubelet.kubeconfig 
- name: create kubelet key 
shell: 
chdir=/root/ssl 
openssl genrsa -out client.key 
- name: create kubelet csr and crt 
shell: 
chdir=/root/ssl 
openssl req -new -key client.key -subj "/CN={{inventory_hostname}}" -out client.csr 
openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -out client.crt 
- name: ´´½¨kubeletµÄsystemd unitÎļþ 
template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service 
- name: ¿ª»úÆôÓÃkubelet ·þÎñ 
shell: systemctl enable kubelet 
ignore_errors: true 
- name: ¿ªÆôkubelet ·þÎñ 
shell: systemctl daemon-reload && systemctl restart kubelet 
- name: create kube-proxy config 
template: src=kube-proxyconfig.j2 dest=/root/kubernetes/kube-proxyconfig 
- name: create kube-proxy systemd unit 
template: src=kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service 
- name: ¿ª»úÆôÓÃkube-proxy ·þÎñ 
shell: systemctl enable kube-proxy 
ignore_errors: true 
- name: ¿ªÆôkube-proxy ·þÎñ 
shell: systemctl daemon-reload && systemctl restart kube-proxy 

//172.16.0.0/16 ÊÇdockerµÄÍø¶Î 
etcdctl set /kubernetes/network/config '{ "Network": "172.16.0.0/16", "Backend": { "Type": "vxlan" } }' 

[root@k8s-master1 tasks]# cat /etc/systemd/system/flanneld.service 
[Unit] 
Description=Flanneld overlay address etcd agent 
After=network-online.target network.target 
Before=docker.service 
[Service] 
Type=notify 
ExecStart=/root/flannel/flanneld \ 
-etcd-endpoints=http://192.168.144.128:2379,http://192.168.144.129:2379 \ 
-etcd-prefix=/kubernetes/network \ 
-iface=ens33 
ExecStartPost=/root/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env 
Restart=on-failure 
[Install] 
WantedBy=multi-user.target 

[Service] 
Type=notify 
EnvironmentFile=/run/flannel/subnet.env 
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock 
ExecReload=/bin/kill -s HUP $MAINPID 

CA 
µ¥ÏòcaÈÏÖ¤ ·þÎñ¶ËÒªÅäÖÃcaÖ¤Ê飬·þÎñ¶ËÖ¤Ê飬˽Կ¡£¿Í»§¶Ë²»Åä»òÕß¾ÍÅäÖÃcaÖ¤Êé 
Ë«ÏòÈÏÖ¤ ·þÎñ¶ËÒªÅäÖÃcaÖ¤Ê飬·þÎñ¶ËÖ¤Ê飬˽Կ¡£ ¿Í»§¶ËÒªÅäÖÃcaÖ¤Ê飬¿Í»§¶Ë¶ÔÓ¦µÄ¿Í»§¶Ë²ÎÊýÒªÅäÖÿͻ§¶Ë˽ԿºÍ¿Í»§¶ËÖ¤Êé 
ÒòΪ˫ÏòÈÏÖ¤·þÎñ¶ËÒ²ÒªÖªµÀÇëÇóµÄÊÇË­ËùÒÔ¿Í»§¶ËÒªÅäÖÃ×Ô¼ºµÄÖ¤ÊéºÍ˽Կ 
pod»áÖؽ¨ ip»á±ä»¯ 
ËùÒÔÎÒÃDz»¿ÉÄÜÔÚ³ÌÐòÖÐÖ±½ÓʹÓÃpodipÀ´µ÷ÓÃpod£¬µ±podʧ°ÜÖØÐÂipµÄ±äÁË¡£ 
service·â×°Á˺ó¶Ë¶à¸öpod£¨Ê¹ÓÃlabel selectÀ´Ñ¡Ôñ¶ÔÓ¦µÄpod£© 
Õâ¸öʹÓÃÎÒÃÇʹÓÃserviceÀ´·ÃÎÊ¡£¡£service»áÉú³ÉÒ»¸öÐéÄâip²»±ä¡£¡£ 
µ±ÎÒÃǵ÷ÓÃÒ»¸öserviceµÄʱºò¡£¡£service¿ÉÒÔÒ»¸ö¸ÅÄî ¾ßÌåÊÇͨ¹ýkube-proxy×é¼þÀ´Íê³É 
kube-proxy»áÔÚ±¾µØ½Úµã½¨Á¢Ò»¸ösocketServerÀ´½ÓÊÜÇëÇó È»ºó½«ÇëÇó·¢Ë͵½ºó¶Ëpod£¨Ê¹ÓÃROund RobinÀ´¸ºÔØ£¬£¬£¬¿ÉÒÔÉèÖÃͬһ¸öip·ÃÎÊijһ¸ö¹Ì¶¨podÀ´±£³Ösession£© 
kube-proxyÔÚÔËÐйý³ÌÖлᶯ̬´´½¨serviceÏà¹ØµÄiptables¹æÔò£¬ 
½«²»¹ÜÊÇ ½Úµãip+nodeport»¹ÊÇ¡¢clusterip+targetport¡¢¡¢ 
µÄÇëÇó×îÖÕÖض¨Ïòµ½¶ÔÓ¦·þÎñµÄ´úÀí¶Ë¿Ú¡£ 
´ÓÉÏÃæÀ´¿´ ÎÒÃÇ¿ÉÒÔÔÚ¼¯ÈºÖÐʹÓÃcluster ipÀ´·ÃÎʶÔÓ¦µÄservice 
µ«ÊÇserviceµÄcluster ipÊÇÓɼ¯Èº·ÖÅäµÄ 
ÕâÑù¾Íµ¼ÖÂÎÒÃDz»ÄÜÔ¤ÏÈÖªµÀip£¬£¬£¬ÒªµÈ½¨Á¢ºÃserviceºó£¬£¬Ê¹ÓÃkubectl get svcÀ´²éѯ 
ËùÒÔÎÒÃÇҪʹÓÃkube-dnsÀ´´¦Àí Ö±½ÓʹÓÃÓòÃûÀ´·ÃÎÊservice 
dnsÊÇÓòÃû½âÎö ½«¶ÔÓ¦µÄÓòÃûת³ÉipµØÖ· 
SkyDNS 
etcd ´æ´¢skydnsÐèÒªµÄÊý¾Ý ±ÈÈçÓòÃûºÍipµÄÓ³Éä 
kube2sky ¼àÌýk8s service×ÊÔ´±ä»¯½«Êý¾ÝдÈëetcdÖÐ £¨Í¨¹ýkubernetesµÄServiceµ÷ÓÃk8sµÄlistºÍwatch API£© 
skydns ¶ÁÈ¡etcdµÄÊý¾Ý£¬ÌṩÓòÃû½âÎöµÄ¹¦ÄÜ 
ÓòÃû¸ñʽ 
<service_name>.<namespace>.svc.<cluster_domain> 
ͨ³£ÎÒÃÇÖ»ÊÇʹÓÃservice_name¾Í¿ÉÒÔµ÷Óà 
ÒòΪnamespaceĬÈÏÊÇdefault 
cluster_domain ÔÚkubelet×é¼þÅäÖà --cluster-domain 
ͬʱҪkube2skyµÄ--domainÒ²ÒªÅäÖà Á½¸öÒªÒ»Ñù 
ΪÁËÔÚPodÖе÷ÓÃÆäËûService£¬kubelet»á×Ô¶¯ÔÚÈÝÆ÷Öд´½¨ÓòÃû½âÎöÅäÖã¨/etc/resolv.conf£© 
ÎÒÏëÎÊÏÂetcdÕâ¸öÈÝÆ÷¿ÉÒÔÊ¡µôÂð£¿k8s¼¯ÈºÓÐetcd²»¿ÉÒÔ¹²ÓÃÂ𣿠
ÀíÂÛÉÏ¿ÉÒÔ¹²ÓÃetcd¡£´Ó¸ôÀëÐԵĽǶȿ¼ÂÇÀ´Ëµ»¹ÊÇ·Ö¿ªºÃ£¬ÕâÑùkube-dns·þÎñ²»»á¶ÔÕû¸ök8s¼¯ÈºµÄÎȶ¨ÐÔ²úÉúÓ°Ïì¡£ÁíÍâÈç¹û°Ñkube-dns¿´×öÒ»¸ö΢·þÎñµÄ»°£¬ÄÇôӦ¸Ã±£Ö¤ÄÚ²¿×é¼þ²»ÒÀÀµÍⲿ£¬¿ÉÒÔ¶ÀÁ¢ÔËÐС£ 

//ÒÑ´¦Àí 
caÖ¤ÊéµÄÀí½â ¼ûÁíһƪskyDns¡£ 
//ÒÑ´¦Àí 
//etcd±¨³öÎÊÌâ--Õâ¸öÊÇetcd°æ±¾ÎÊÌâ,²»Ó°ÏìʹÓÃ,ÎÒÊÇyum install etcd¸ãµÄ 
//²Î¿¼https://www.cnblogs.com/iiiiher/p/7879587.html 
watch chan error: etcdserver: mvcc: required revision has been compacted 
//ÒÑ´¦Àí 
//pause¾µÏñÀ­È¡²»ÏÂÀ´ 
docker pull mirrorgooglecontainers/pause:3.1 
//¸Ä³É×Ô¼ºµÄ xujinding:3.1ÔÚkubeletÆô¶¯²ÎÊýÖÐÅäÖà 
docker tag mirrorgooglecontainers/pause:3.1 xujinding:3.1 
//δ´¦Àí 
//kube-controller-manager.ERROR 
no cloud provider provided, services of type LoadBalancer will fail 
kubelet.ERROR 
E0523 11:04:06.491945 9040 kubelet.go:1252] Image garbage collection failed once. Stats initialization may not have completed yet: failed to get imageFs info: unable to find data for container / 
//ÒÑ´¦Àí 
ʹÓÃkubectl get node/pod ûÎÊÌâ µ«ÊÇʹÓà log pod¡ªname¾ÍÓгöÏÖÏÂÃæ´íÎó ¼´Ê¹Ê¹ÓÃÁËÈçÏÂÃæ´øÖ¤ÊéµÄÃüÁîÒ²»á 
[root@k8s-master1 ~]# kubectl --server=https://192.168.144.128:6443 --certificate-authority=/root/ssl/ca.crt --client-certificate=/root/ssl/client.crt --client-key=/root/ssl/client.key log mysql-rc-9zgcp 
error: You must be logged in to the server (the server has asked for the client to provide credentials ( pods/log mysql-rc-9zgcp)) 
Ô­ÒòÊÇÔÚkubelet.service ½«ÄäÃûµÇ¼¹ØÁË£¨--anonymous-auth=false£© 
¸Ä³Étrue¾ÍÐÐÁË 
ÒÑ´¦Àí 
[root@k8s-master2 ~]# kubectl exec -it mysql-rc-v2-l8wtj /bin/sh 
error: unable to upgrade connection: Unauthorized 
µÇ¼²»ÁËÈÝÆ÷ 
Ô­Òò£ºÍ¬ÉÏÒ»¸öÎÊÌâ 
ÒÉÎÊ£¿£¿ 
¼ÙÉèÎÒÔÚ»úÆ÷128 ½¨Ò»¸öservice ²¢Ö´ÐÐ Ó³Éä¶Ë¿Ú33060 
Ϊʲô ʹÓÃ129µÄ»úÆ÷µÄipÒ²¿ÉÒÔ·ÃÎÊ 
//δ´¦Àí 
ansibleÖ´ÐкÜÂýÂý 
//δ´¦Àí 
´Óansible hosts¶ÁÈ¡±äÁ¿ÎÊÌâ 
all: 
hosts: 
49.108.221.144: 
api_node_name: "ddddddddddddddd" 
ansible_port: 22 
ansible_user: root 
ansible_ssh_pass: root 
blog: 
ansible_host: 49.108.221.144 
ansible_port: 22 
ansible_user: root 
ansible_ssh_pass: root 
children: 
node_testiii: 
hosts: 
39.108.231.124: 
api_node_name: "haha124124" 
49.108.221.144: 
api_node_name: "haha14444444" 
node_test: 
hosts: 
39.108.231.124: 
api_node_name: "haha22221241241234" 
49.108.221.144: 
api_node_name: "haha22224444444444444" 
ansible ´òÓ¡api_node_name£¿£¿Êä³öʲô£¿£¿ 
--- 
- hosts: node_test 
tasks: 
- name: test 
debug: 
msg: "{{ api_node_name }}" 
- name: 
debug: 
msg: "{{ api_node_name.second }}" 
- name: jjj 
debug: 
msg: "dddd" 
¸÷ÖÖÇé¿ö ÒѾ­ÃɱΠ
ËùÒÔʵ¼ÊÉÏÏÈÃüÃû²»Í¬