Skip to content

Latest commit

 

History

History
240 lines (229 loc) · 8.43 KB

09-1.calico网络设置.md

File metadata and controls

240 lines (229 loc) · 8.43 KB

9-1 核心组件安装之calico

安装前准备

确保 Calico 可以管理主机上的 cali 和 tunl 接口。 如果主机上存在NetworkManager，请按照下面方法配置 NetworkManager。 NetworkManager 管理默认网络命名空间中接口的路由表的功能，可能会干扰 Calico 正确处理网络路由的能力。

创建一个配置文件/etc/NetworkManager/conf.d/calico.conf，来制止这种干扰：

[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*

关于 NetworkManager 配置请参考 官方链接；

创建证书和签名

$ cd /root/pki/ssl
$ cat > calico-csr.json <<EOF
{
  "CN": "calico",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

生成证书和私钥

$ cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
  -ca-key=/etc/kubernetes/ssl/ca-key.pem \
  -config=/etc/kubernetes/ssl/ca-config.json \
  -profile=kubernetes calico-csr.json | cfssljson -bare calico

$ ls calico*pem
$ chmod 755 /etc/calico/ssl/calico*pem

将生成的证书和私钥分发到所有节点（master 和 worker）

$ export NODE_IPS_all=(192.168.133.129 192.168.133.130 192.168.133.131 192.168.133.132)
$ for node_ip in ${NODE_IPS_all[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "mkdir -p /etc/calico/ssl"
    scp calico* ${node_ip}:/etc/calico/ssl
    ssh ${node_ip} "chmod 755 /etc/calico/ssl/calico*pem"
  done

下载相关文件

$ curl https://docs.projectcalico.org/v3.5/getting-started/kubernetes/installation/hosted/calico.yaml -O

创建 calico-etcd-secrets

$ kubectl create secret generic -n kube-system calico-etcd-secrets \
        --from-file=etcd-ca=/etc/kubernetes/ssl/ca.pem \
        --from-file=etcd-key=/etc/calico/ssl/calico-key.pem \
        --from-file=etcd-cert=/etc/calico/ssl/calico.pem

提示： 如果手动创建了 calico-etcd-secrets内容，那么就需要将calico.yaml文件中的Secret注释或者删除掉。如果不执行这一步，那么可以通过如下方式来实现：

$ export ETCD_CERT=`cat /etc/calico/ssl/calico.pem | base64 | tr -d '\n'`
$ export ETCD_KEY=`cat /etc/calico/ssl/calico-key.pem | base64 | tr -d '\n'`
$ export ETCD_CA=`cat /etc/kubernetes/ssl/ca.pem | base64 | tr -d '\n'`

$ sed -i "s@.*etcd-cert:.*@\ \ etcd-cert:\ ${ETCD_CERT}@gi" calico.yaml
$ sed -i "s@.*etcd-key:.*@\ \ etcd-key:\ ${ETCD_KEY}@gi" calico.yaml
$ sed -i "s@.*etcd-ca:.*@\ \ etcd-ca:\ ${ETCD_CA}@gi" calico.yaml

修改calico.yml文件

## 更改为自己的etcd集群
$ sed -i 's@.*etcd_endpoints:.*@\ \ etcd_endpoints:\ \"https://192.168.133.128:2379,https://192.168.133.129:2379,https://192.168.133.130:2379\"@gi' calico.yaml

## 通过证书访问etcd集群
$ sed -i 's@__ETCD_KEY_FILE__@/etc/calico/ssl/calico-key.pem@gi' calico.yaml
$ sed -i 's@__ETCD_CERT_FILE__@/etc/calico/ssl/calico.pem@gi' calico.yaml
$ sed -i 's@__ETCD_CA_CERT_FILE__@/etc/kubernetes/ssl/ca.pem@gi' calico.yaml
 
$ sed -i 's@.*etcd_ca:.*@\ \ etcd_ca:\ "/calico-secrets/etcd-ca"@gi' calico.yaml
$ sed -i 's@.*etcd_cert:.*@\ \ etcd_cert:\ "/calico-secrets/etcd-cert"@gi' calico.yaml
$ sed -i 's@.*etcd_key:.*@\ \ etcd_key:\ "/calico-secrets/etcd-key"@gi' calico.yaml

## 修改 pod IP网段
$ sed -i '[email protected]/[email protected]/16@gi' calico.yaml

如何选择 IPIP与BGP模式

# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
  value: "Always

提示： 该内容在calico.yaml文件中，通过搜索就能找到。如果设置为Always表示开启IPIP，设置为off表示为启用BGP。

运行 calico pod

$ export NODE_IPS_all=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS_all[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "rm -rf /etc/cni/net.d/*"
  done
  或者
$ for node_ip in ${NODE_IPS_all[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "mv /etc/cni/net.d/10-default.conf /etc/cni/net.d/10-default.conf.bak"
  done
$ kubectl create -f rbac.yaml                     rbac.yaml在这个版本中已经集成到calico.yaml中了，执行下面一个yaml就可以了。
$ kubectl create -f calico.yaml

安装 calicoctl 管理命令

$ wget https://github.com/projectcalico/calicoctl/releases/download/v3.5.4/calicoctl
$ chmod +x calicoctl
$ ./calicoctl node status
Calico process is running.

IPv4 BGP status
+-----------------+-------------------+-------+----------+-------------+
|  PEER ADDRESS   |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+-----------------+-------------------+-------+----------+-------------+
| 192.168.133.129 | node-to-node mesh | up    | 13:35:33 | Established |
| 192.168.133.130 | node-to-node mesh | up    | 13:35:34 | Established |
| 192.168.133.131 | node-to-node mesh | up    | 13:35:34 | Established |
+-----------------+-------------------+-------+----------+-------------+

IPv6 BGP status
No IPv6 peers found.

BGP 协议是通过TCP 连接来建立邻居的，因此可以用netstat 命令验证 BGP Peer

$ netstat -antlp|grep ESTABLISHED|grep 179
tcp        0      0 192.168.133.128:13226      192.168.133.128:179        ESTABLISHED 84068/bird
tcp        0      0 192.168.133.128:31565      192.168.133.129:179        ESTABLISHED 84068/bird
tcp        0      0 192.168.133.128:179        192.168.133.131:53066      ESTABLISHED 84068/bird
tcp        0      0 192.168.133.128:179        192.168.133.132:22586      ESTABLISHED 84068/bird
tcp        0      0 192.168.133.128:5613       192.168.133.130:179        ESTABLISHED 84068/bird

启动相关容器pod验证网络服务

$ cat > nginx-ds.yml <<EOF
apiVersion: v1
kind: Service
metadata:
  name: nginx-ds
  labels:
    app: nginx-ds
spec:
  type: NodePort
  selector:
    app: nginx-ds
  ports:
  - name: http
    port: 80
    targetPort: 80
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nginx-ds
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  template:
    metadata:
      labels:
        app: nginx-ds
    spec:
      containers:
      - name: my-nginx
        image: nginx:1.7.9
        ports:
        - containerPort: 80
EOF

$ kubectl create -f nginx-ds.yml
$ kubectl get pods  -o wide|grep nginx-ds

检查Node与Pod IP连通性

$ kubectl get pods  -o wide|grep nginx-ds
nginx-ds-2z4gv   1/1     Running   0          21s   172.30.181.193   192.168.133.131   <none>           <none>
nginx-ds-flzrd   1/1     Running   0          21s   172.30.217.1     192.168.133.129   <none>           <none>
nginx-ds-kqfrw   1/1     Running   0          21s   172.30.205.129   192.168.133.130   <none>           <none>
nginx-ds-xjbb5   1/1     Running   0          21s   172.30.176.1     192.168.133.128   <none>           <none>

ping 每个节点上运行的Pod

$ NODE_IPS_all=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS_all[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "ping -c 4 172.30.181.193"
    ssh ${node_ip} "ping -c 4 172.30.217.1"
    ssh ${node_ip} "ping -c 4 172.30.205.129"
    ssh ${node_ip} "ping -c 4 172.30.176.1"
  done

检查Service IP 和 端口可达性

$ kubectl get services |grep nginx-ds
nginx-ds     NodePort    10.254.158.126   <none>        80:30518/TCP   3m51s

$ NODE_IPS_all=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS_all[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "curl 10.254.158.126"
  done

查看etcd中calico相关信息

因为这里calico网络使用etcd存储数据，所以可以在etcd集群中查看数据。calico 3.x 版本默认使用 etcd v3存储，登陆集群的一个etcd 节点，查看命令： 查看所有calico相关数据

$ export ETCDCTL_API=3
$ etcdctl \
  --cacert=/etc/kubernetes/ssl/ca.pem \
  --cert=/etc/etcd/ssl/etcd.pem \
  --key=/etc/etcd/ssl/etcd-key.pem \
  --endpoints="https://192.168.133.128:2379,https://192.168.133.129:2379,https://192.168.133.130:2379" \
  get --prefix /calico

查看 calico网络为各节点分配的网段

$ export ETCDCTL_API=3
$ etcdctl \
  --cacert=/etc/kubernetes/ssl/ca.pem \
  --cert=/etc/etcd/ssl/etcd.pem \
  --key=/etc/etcd/ssl/etcd-key.pem \
  --endpoints="https://192.168.133.128:2379,https://192.168.133.129:2379,https://192.168.133.130:2379" \
  get --prefix /calico/ipam/v2/host