也是基于cni网络,
- 1.替代了kube-proxy组件,无需在部署kube-router,解决了svc网络
- 2.自带cni,bgp,解决了pod网络
- 3.基于ipvs转发
- 4.路由传播依赖bgp
kuberouter结构
参考(部署步骤): https://cloudnativelabs.github.io/post/2017-04-19-kube-router/
部署步骤
要注意的是
- 1./root/bootstrap.kubeconfig文件
2.节点开启支持ipv6
- kubelet 要有--network-plugin-dir=/opt/cni/bin --network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --allow-privileged=true
环境准备
etcdctl set /kubernetes/network/config < flannel-config.json
kube-apiserver --service-cluster-ip-range=10.254.0.0/16 --etcd-servers=http://127.0.0.1:2379 --insecure-bind-address=0.0.0.0 --admission-control=ServiceAccount --service-account-key-file=/root/ssl/ca.key --client-ca-file=/root/ssl/ca.crt --tls-cert-file=/root/ssl/server.crt --tls-private-key-file=/root/ssl/server.key --allow-privileged=true --storage-backend=etcd2 --v=2 --enable-bootstrap-token-auth --token-auth-file=/root/token.csv
kube-controller-manager --master=http://127.0.0.1:8080 --service-account-private-key-file=/root/ssl/ca.key --cluster-signing-cert-file=/root/ssl/ca.crt --cluster-signing-key-file=/root/ssl/ca.key --root-ca-file=/root/ssl/ca.crt --v=2 --allocate-node-cidrs=true --cluster-cidr=10.1.0.0/16
kube-scheduler --master=http://127.0.0.1:8080 --v=2
kubelet --allow-privileged=true --cluster-dns=10.254.0.2 --cluster-domain=cluster.local --v=2 --experimental-bootstrap-kubeconfig=/root/bootstrap.kubeconfig --kubeconfig=/root/kubelet.kubeconfig --fail-swap-on=false --network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --allow-privileged=true
kube-proxy --master=http://192.168.14.11:8080 --v=2准备token.csv和bootstrap.kubeconfig文件
- 在master生成token.csv
BOOTSTRAP_TOKEN="41f7e4ba8b7be874fcff18bf5cf41a7c"
cat > token.csv<<EOF
41f7e4ba8b7be874fcff18bf5cf41a7c,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
- 将bootstrap.kubeconfig同步到所有节点
设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=/root/ssl/ca.crt --embed-certs=true --server=http://192.168.14.11:8080 --kubeconfig=bootstrap.kubeconfig
设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap --token="41f7e4ba8b7be874fcff18bf5cf41a7c" --kubeconfig=bootstrap.kubeconfig
设置上下文参数
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig
设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
两个文件我都放在了/root下.这里用到bootstrap.kubeconfig,同步到node各个节点.
部署kube-router
[root@n1 kube-router]# cat kube-router.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-router-cfg
namespace: kube-system
labels:
tier: node
k8s-app: kube-router
data:
cni-conf.json: |
{
"name":"kubernetes",
"type":"bridge",
"bridge":"kube-bridge",
"isDefaultGateway":true,
"ipam": {
"type":"host-local"
}
}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: kube-router
namespace: kube-system
labels:
k8s-app: kube-router
spec:
template:
metadata:
labels:
k8s-app: kube-router
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
containers:
- name: kube-router
image: cloudnativelabs/kube-router
args: ["--run-router=true", "--run-firewall=true", "--run-service-proxy=true", "--kubeconfig=/var/lib/kube-router/kubeconfig"]
securityContext:
privileged: true
imagePullPolicy: Always
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: cni-conf-dir
mountPath: /etc/cni/net.d
- name: kubeconfig
mountPath: /var/lib/kube-router/kubeconfig
readOnly: true
initContainers:
- name: install-cni
image: busybox
imagePullPolicy: Always
command:
- /bin/sh
- -c
- set -e -x;
if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
cp /etc/kube-router/cni-conf.json ${TMP};
mv ${TMP} /etc/cni/net.d/10-kuberouter.conf;
fi
volumeMounts:
- name: cni-conf-dir
mountPath: /etc/cni/net.d
- name: kube-router-cfg
mountPath: /etc/kube-router
hostNetwork: true
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
volumes:
- name: lib-modules
hostPath:
path: /lib/modules
- name: cni-conf-dir
hostPath:
path: /etc/cni/net.d
- name: kube-router-cfg
configMap:
name: kube-router-cfg
- name: kubeconfig
hostPath:
path: /root/bootstrap.kubeconfig
注: /root/bootstrap.kubeconfig.
[root@n1 kube-router]# kk
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE LABELS
kube-system kube-router-989p5 1/1 Running 0 9m 192.168.14.12 n2.ma.com controller-revision-hash=1689399381,k8s-app=kube-router,pod-template-generation=1
kube-system kube-router-plmpv 1/1 Running 0 9m 192.168.14.13 n3.ma.com controller-revision-hash=1689399381,k8s-app=kube-router,pod-template-generation=1测试连通性
kubectl run -it --rm --restart=Never b10 --image=busybox sh
kubectl run -it --rm --restart=Never b20 --image=busybox sh
[root@n1 ~]# kk
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE LABELS
default b10 1/1 Running 0 16s 10.1.1.26 n3.ma.com run=b10
default b20 1/1 Running 0 7s 10.1.0.14 n2.ma.com run=b20
[root@n1 yaml]# kubectl run -it --rm --restart=Never b10 --image=busybox sh
If you don't see a command prompt, try pressing enter.
/ # ping 10.1.0.14
PING 10.1.0.14 (10.1.0.14): 56 data bytes
64 bytes from 10.1.0.14: seq=0 ttl=62 time=2.018 ms
64 bytes from 10.1.0.14: seq=1 ttl=62 time=0.576 ms
^C遇到的问题
- 1./root/bootstrap.kubeconfig文件
2.节点开启支持ipv6
- kubelet 要有--network-plugin-dir=/opt/cni/bin --network-plugin=cni --cni-conf-dir=/etc/cni/net.d/ --allow-privileged=true
