From 1d5b364e8c3ba33d71c97f1cf7d2984012795c84 Mon Sep 17 00:00:00 2001 From: pixiake Date: Tue, 21 Apr 2020 18:25:31 +0800 Subject: [PATCH] add K2Cluster object --- apis/v1alpha1/cluster.go | 8 + apis/v1alpha1/host.go | 18 +- apis/v1alpha1/register.go | 42 ++ cluster/kubernetes/master.go | 205 ++++++++ cluster/kubernetes/nodes.go | 4 +- cluster/preinstall/initOS.go | 21 +- cmd/create/config.go | 67 +++ install/install.go | 4 + plugins/network/calico/tmpl.go | 818 ++++++++++++++++++++++++++++++++ plugins/network/flannel/tmpl.go | 620 ++++++++++++++++++++++++ plugins/network/network.go | 77 +++ util/runner/runner.go | 4 +- 12 files changed, 1867 insertions(+), 21 deletions(-) create mode 100644 apis/v1alpha1/register.go create mode 100644 cmd/create/config.go create mode 100644 plugins/network/calico/tmpl.go create mode 100644 plugins/network/flannel/tmpl.go create mode 100644 plugins/network/network.go diff --git a/apis/v1alpha1/cluster.go b/apis/v1alpha1/cluster.go index e90fa08b..aa9183d9 100644 --- a/apis/v1alpha1/cluster.go +++ b/apis/v1alpha1/cluster.go @@ -3,6 +3,7 @@ package v1alpha1 import ( "fmt" "github.com/pixiake/kubekey/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -31,6 +32,13 @@ const ( WorkerRole = "worker" ) +type K2Cluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec ClusterCfg `json:"spec"` + //Status ClusterStatus `json:"status"` +} + type ClusterCfg struct { Hosts []HostCfg `yaml:"hosts" json:"hosts,omitempty"` LBKubeApiserver LBKubeApiserverCfg `yaml:"lbKubeapiserver" json:"lbKubeapiserver,omitempty"` diff --git a/apis/v1alpha1/host.go b/apis/v1alpha1/host.go index f727f5df..1f150ec0 100644 --- a/apis/v1alpha1/host.go +++ b/apis/v1alpha1/host.go @@ -1,19 +1,19 @@ package v1alpha1 type HostCfg struct { - HostName string `yaml:"hostName,omitempty" json:"hostName,omitempty"` + HostName string `yaml:"hostName" json:"hostName,omitempty"` SSHAddress string `yaml:"sshAddress" json:"sshAddress,omitempty"` InternalAddress string `yaml:"internalAddress" json:"internalAddress,omitempty"` Port string `yaml:"port" json:"port,omitempty"` User string `yaml:"user" json:"user,omitempty"` - Password string `yaml:"password" json:"password,omitempty"` - SSHKeyPath string `yaml:"sshKeyPath" json:"sshKeyPath,omitempty"` - Role []string `yaml:"role" json:"role,omitempty" norman:"type=array[enum],options=etcd|master|worker"` - ID int `json:"-"` - IsEtcd bool - IsMaster bool - IsWorker bool - OSFamily string + Password string `yaml:"password, omitempty" json:"password,omitempty"` + SSHKeyPath string `yaml:"sshKeyPath, omitempty" json:"sshKeyPath,omitempty"` + Role []string `yaml:"role" json:"role,omitempty" norman:"type=array[enum],options=etcd|master|worker|client"` + ID int `yaml:"omitempty" json:"-"` + IsEtcd bool `yaml:"omitempty"` + IsMaster bool `yaml:"omitempty"` + IsWorker bool `yaml:"omitempty"` + IsClient bool `yaml:"omitempty"` } type Hosts struct { diff --git a/apis/v1alpha1/register.go b/apis/v1alpha1/register.go new file mode 100644 index 00000000..3c2ef57b --- /dev/null +++ b/apis/v1alpha1/register.go @@ -0,0 +1,42 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the name of the group used by this API +const GroupName = "kubekey.io" + +// SchemeGroupVersion is group version used to register API objects +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} + +var ( + // SchemeBuilder points to a list of functions added to Scheme + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + // AddToScheme applies all the stored functions to the Scheme + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) +} + +// Kind takes an unqualified kind and returns GroupKind +func Kind(kind string) schema.GroupKind { + return SchemeGroupVersion.WithKind(kind).GroupKind() +} + +// Resource takes an unqualified resource and returns GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &ClusterCfg{}) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/cluster/kubernetes/master.go b/cluster/kubernetes/master.go index 1a5ea9e8..a549252e 100644 --- a/cluster/kubernetes/master.go +++ b/cluster/kubernetes/master.go @@ -8,6 +8,18 @@ import ( "github.com/pixiake/kubekey/util/manager" "github.com/pixiake/kubekey/util/ssh" "github.com/pkg/errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +var ( + joinMasterCmd = "" + joinWorkerCmd = "" + clusterInfo = "" + kubeConfig = "" ) func InitKubernetesCluster(mgr *manager.Manager) error { @@ -27,7 +39,200 @@ func initKubernetesCluster(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn if err1 != nil { return errors.Wrap(errors.WithStack(err1), "failed to generate kubeadm config") } + + output, err2 := mgr.Runner.RunCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml\"") + if err2 != nil { + fmt.Println(output) + return errors.Wrap(errors.WithStack(err2), "failed to init kubernetes cluster") + } + err3 := GetKubeConfig(mgr) + if err3 != nil { + return err3 + } + err4 := removeMasterTaint(mgr, node) + if err4 != nil { + return err4 + } + err5 := addWorkerLabel(mgr, node) + if err5 != nil { + return err5 + } } return nil } + +func GetKubeConfig(mgr *manager.Manager) error { + createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" + getKubeConfigCmd := "cp -f /etc/kubernetes/admin.conf /root/.kube/config" + getKubeConfigCmdUsr := "cp -f /etc/kubernetes/admin.conf $HOME/.kube/config" + chownKubeConfig := "chown $(id -u):$(id -g) $HOME/.kube/config" + + cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd, getKubeConfigCmdUsr, chownKubeConfig}, " && ") + _, err := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", cmd)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to init kubernetes cluster") + } + return nil +} + +func removeMasterTaint(mgr *manager.Manager, node *kubekeyapi.HostCfg) error { + if node.IsWorker { + removeMasterTaintCmd := fmt.Sprintf("/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=:NoSchedule-", node.HostName) + _, err := mgr.Runner.RunCmd(removeMasterTaintCmd) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to remove master taint") + } + } + return nil +} + +func addWorkerLabel(mgr *manager.Manager, node *kubekeyapi.HostCfg) error { + if node.IsWorker { + addWorkerLabelCmd := fmt.Sprintf("/usr/local/bin/kubectl label node %s node-role.kubernetes.io/worker=", node.HostName) + out, err := mgr.Runner.RunCmd(addWorkerLabelCmd) + if err != nil && !strings.Contains(out, "already") { + return errors.Wrap(errors.WithStack(err), "failed to add worker label") + } + } + return nil +} + +func GetJoinNodesCmd(mgr *manager.Manager) error { + mgr.Logger.Infoln("Get join nodes cmd") + + return mgr.RunTaskOnMasterNodes(getJoinNodesCmd, false) +} + +func getJoinNodesCmd(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + getJoinCmd(mgr) + return nil +} + +func getJoinCmd(mgr *manager.Manager) error { + uploadCertsCmd := "/usr/local/bin/kubeadm init phase upload-certs --upload-certs" + out, err := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", uploadCertsCmd)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to upload kubeadm certs") + } + reg := regexp.MustCompile("[0-9|a-z]{64}") + certificateKey := reg.FindAllString(out, -1)[0] + err1 := PatchKubeadmSecret(mgr) + if err1 != nil { + return err1 + } + + tokenCreateMasterCmd := fmt.Sprintf("/usr/local/bin/kubeadm token create --print-join-command --certificate-key %s", certificateKey) + + out, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", tokenCreateMasterCmd)) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to get join node cmd") + } + + joinMasterStrList := strings.Split(out, "kubeadm join") + joinMasterStr := strings.Split(joinMasterStrList[1], certificateKey) + joinMasterCmd = fmt.Sprintf("/usr/local/bin/kubeadm join %s %s", joinMasterStr[0], certificateKey) + + joinWorkerStrList := strings.Split(joinMasterCmd, "--control-plane") + joinWorkerCmd = joinWorkerStrList[0] + + out, err3 := mgr.Runner.RunCmd("/usr/local/bin/kubectl get nodes -o wide") + if err3 != nil { + return errors.Wrap(errors.WithStack(err3), "failed to get cluster info") + } + clusterInfo = out + + kubeCfgBase64Cmd := "cat /etc/kubernetes/admin.conf | base64 --wrap=0" + out, err4 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", kubeCfgBase64Cmd)) + if err4 != nil { + return errors.Wrap(errors.WithStack(err4), "failed to get cluster kubeconfig") + } + kubeConfig = out + + currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return errors.Wrap(err, "faild get current dir") + } + exec.Command(fmt.Sprintf("mkdir -p %s/kubekey", currentDir)) + exec.Command(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > %s/kubekey/kubeconfig.yaml\"", kubeConfig, currentDir)).Run() + + return nil +} + +func PatchKubeadmSecret(mgr *manager.Manager) error { + externalEtcdCerts := []string{"external-etcd-ca.crt", "external-etcd.crt", "external-etcd.key"} + for _, cert := range externalEtcdCerts { + _, err := mgr.Runner.RunCmd(fmt.Sprintf("/usr/local/bin/kubectl patch -n kube-system secret kubeadm-certs -p '{\"data\": {\"%s\": \"\"}}'", cert)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to patch kubeadm secret") + } + } + return nil +} + +func JoinNodesToCluster(mgr *manager.Manager) error { + mgr.Logger.Infoln("Join nodes to cluster") + + return mgr.RunTaskOnK8sNodes(joinNodesToCluster, true) +} + +func joinNodesToCluster(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + if !strings.Contains(clusterInfo, node.HostName) && !strings.Contains(clusterInfo, node.InternalAddress) { + if node.IsMaster { + err := addMaster(mgr) + if err != nil { + return err + } + err1 := removeMasterTaint(mgr, node) + if err1 != nil { + return err1 + } + err2 := addWorkerLabel(mgr, node) + if err2 != nil { + return err2 + } + } + if node.IsWorker && !node.IsMaster { + err := addWorker(mgr) + if err != nil { + return err + } + err1 := addWorkerLabel(mgr, node) + if err1 != nil { + return err1 + } + } + } + return nil +} + +func addMaster(mgr *manager.Manager) error { + _, err := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", joinMasterCmd)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to add master to cluster") + } + err1 := GetKubeConfig(mgr) + if err1 != nil { + return err1 + } + return nil +} + +func addWorker(mgr *manager.Manager) error { + _, err := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", joinWorkerCmd)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to add worker to cluster") + } + createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" + chownKubeConfig := "chown $(id -u):$(id -g) $HOME/.kube/config" + _, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd)) + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "failed to create kube dir") + } + syncKubeconfigCmd := fmt.Sprintf("echo %s | base64 -d > %s && echo %s | base64 -d > %s && %s", kubeConfig, "/root/.kube/config", kubeConfig, "$HOME/.kube/config", chownKubeConfig) + _, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", syncKubeconfigCmd)) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to sync kube config") + } + return nil +} diff --git a/cluster/kubernetes/nodes.go b/cluster/kubernetes/nodes.go index 788a60ff..e671b149 100644 --- a/cluster/kubernetes/nodes.go +++ b/cluster/kubernetes/nodes.go @@ -47,7 +47,7 @@ func syncKubeBinaries(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.C if strings.Contains(binary, "cni-plugins-linux") { cmdlist = append(cmdlist, fmt.Sprintf("mkdir -p /opt/cni/bin && tar -zxf %s/%s -C /opt/cni/bin", "/tmp/kubekey", binary)) } else { - cmdlist = append(cmdlist, fmt.Sprintf("cp /tmp/kubekey/%s /usr/local/bin/%s && chmod +x /usr/local/bin/%s", binary, strings.Split(binary, "-")[0], strings.Split(binary, "-")[0])) + cmdlist = append(cmdlist, fmt.Sprintf("cp -f /tmp/kubekey/%s /usr/local/bin/%s && chmod +x /usr/local/bin/%s", binary, strings.Split(binary, "-")[0], strings.Split(binary, "-")[0])) } } cmd := strings.Join(cmdlist, " && ") @@ -70,7 +70,7 @@ func setKubelet(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connect return err1 } kubeletServiceBase64 := base64.StdEncoding.EncodeToString([]byte(kubeletService)) - _, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/systemd/system/kubelet.service && ln -snf /usr/local/bin/kubelet /usr/bin/kubelet\"", kubeletServiceBase64)) + _, err2 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/systemd/system/kubelet.service && systemctl enable kubelet && ln -snf /usr/local/bin/kubelet /usr/bin/kubelet\"", kubeletServiceBase64)) if err2 != nil { return errors.Wrap(errors.WithStack(err2), "failed to generate kubelet service") } diff --git a/cluster/preinstall/initOS.go b/cluster/preinstall/initOS.go index d585664d..47c99dc8 100644 --- a/cluster/preinstall/initOS.go +++ b/cluster/preinstall/initOS.go @@ -23,20 +23,25 @@ func initOsOnNode(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Conne return errors.Wrap(errors.WithStack(err), "failed to init operating system") } - initOsScript, err1 := tmpl.InitOsScript(mgr.Cluster) + _, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"hostnamectl set-hostname %s && sed -i '/^127.0.1.1/s/.*/127.0.1.1 %s/g' /etc/hosts\"", node.HostName, node.HostName)) if err1 != nil { - return err1 + return errors.Wrap(errors.WithStack(err1), "failed to override hostname") + } + + initOsScript, err2 := tmpl.InitOsScript(mgr.Cluster) + if err2 != nil { + return err2 } str := base64.StdEncoding.EncodeToString([]byte(initOsScript)) - _, err2 := mgr.Runner.RunCmd(fmt.Sprintf("echo %s | base64 -d > %s/initOS.sh && chmod +x %s/initOS.sh", str, tmpDir, tmpDir)) - if err2 != nil { - return errors.Wrap(errors.WithStack(err2), "failed to init operating system") - } - - _, err3 := mgr.Runner.RunCmd(fmt.Sprintf("sudo %s/initOS.sh", tmpDir)) + _, err3 := mgr.Runner.RunCmd(fmt.Sprintf("echo %s | base64 -d > %s/initOS.sh && chmod +x %s/initOS.sh", str, tmpDir, tmpDir)) if err3 != nil { return errors.Wrap(errors.WithStack(err3), "failed to init operating system") } + + _, err4 := mgr.Runner.RunCmd(fmt.Sprintf("sudo %s/initOS.sh", tmpDir)) + if err4 != nil { + return errors.Wrap(errors.WithStack(err4), "failed to init operating system") + } return nil } diff --git a/cmd/create/config.go b/cmd/create/config.go new file mode 100644 index 00000000..7d6ebeb3 --- /dev/null +++ b/cmd/create/config.go @@ -0,0 +1,67 @@ +package create + +import ( + "encoding/base64" + "fmt" + "github.com/lithammer/dedent" + "github.com/pixiake/kubekey/util" + "github.com/pixiake/kubekey/util/manager" + "github.com/pkg/errors" + "os" + "os/exec" + "path/filepath" + "text/template" +) + +var ( + K2ClusterObjTempl = template.Must(template.New("etcdSslCfg").Parse( + dedent.Dedent(`apiVersion: kubekey.io/v1alpha1 +kind: K2Cluster +metadata: + name: demo +spec: + hosts: + - hostName: node1 + sshAddress: 172.16.0.2 + internalAddress: 172.16.0.2 + port: "22" + user: ubuntu + password: Qcloud@123 + sshKeyPath: "" + role: + - etcd + - master + - worker + lbKubeapiserver: + domain: lb.kubesphere.local + address: "" + port: "6443" + kubeVersion: v1.17.4 + kubeImageRepo: kubekey + kubeClusterName: cluster.local + network: + plugin: calico + kube_pods_cidr: 10.233.64.0/18 + kube_service_cidr: 10.233.0.0/18 + `))) +) + +func GenerateK2ClusterObjStr(mgr *manager.Manager, index int) (string, error) { + return util.Render(K2ClusterObjTempl, util.Data{}) +} + +func GenerateK2ClusterObj() error { + K2ClusterObjStr, _ := GenerateK2ClusterObjStr(nil, 0) + K2ClusterObjStrBase64 := base64.StdEncoding.EncodeToString([]byte(K2ClusterObjStr)) + + currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return errors.Wrap(err, "faild get current dir") + } + cmd := fmt.Sprintf("echo %s | base64 -d > %s/k2cluster-demo.yaml", K2ClusterObjStrBase64, currentDir) + err1 := exec.Command("/bin/sh", "-c", cmd).Run() + if err1 != nil { + return err1 + } + return nil +} diff --git a/install/install.go b/install/install.go index 7bbb7629..3143b819 100644 --- a/install/install.go +++ b/install/install.go @@ -5,6 +5,7 @@ import ( "github.com/pixiake/kubekey/cluster/etcd" "github.com/pixiake/kubekey/cluster/kubernetes" "github.com/pixiake/kubekey/cluster/preinstall" + "github.com/pixiake/kubekey/plugins/network" "github.com/pixiake/kubekey/util/manager" "github.com/pixiake/kubekey/util/task" "github.com/pkg/errors" @@ -20,6 +21,9 @@ func ExecTasks(mgr *manager.Manager) error { {Fn: etcd.GenerateEtcdService, ErrMsg: "failed to start etcd cluster"}, {Fn: kubernetes.ConfigureKubeletService, ErrMsg: "failed to sync kube binaries"}, {Fn: kubernetes.InitKubernetesCluster, ErrMsg: "failed to init kubernetes cluster"}, + {Fn: network.DeployNetworkPlugin, ErrMsg: "failed to deploy network plugin"}, + {Fn: kubernetes.GetJoinNodesCmd, ErrMsg: "failed to get join cmd"}, + {Fn: kubernetes.JoinNodesToCluster, ErrMsg: "failed to join node"}, } for _, step := range createTasks { diff --git a/plugins/network/calico/tmpl.go b/plugins/network/calico/tmpl.go new file mode 100644 index 00000000..e570fa76 --- /dev/null +++ b/plugins/network/calico/tmpl.go @@ -0,0 +1,818 @@ +package calico + +import ( + "github.com/lithammer/dedent" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/util" + "text/template" +) + +var calicoTempl = template.Must(template.New("kubeadmCfg").Parse( + dedent.Dedent(`--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use + veth_mtu: "1440" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } + +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMBlock + plural: ipamblocks + singular: ipamblock + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BlockAffinity + plural: blockaffinities + singular: blockaffinity + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMHandle + plural: ipamhandles + singular: ipamhandle + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMConfig + plural: ipamconfigs + singular: ipamconfig + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkSet + plural: networksets + singular: networkset +--- +# Source: calico/templates/rbac.yaml + +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: + - kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only requried for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: + - kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: calico/cni:v3.13.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.13.0 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: calico/pod2daemon-flexvol:v3.13.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: calico/node:v3.13.0 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within --cluster-cidr. + - name: CALICO_IPV4POOL_CIDR + value: "{{ .KubePodsCIDR }}" + # Disable file logging so kubectl logs works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + +--- +# Source: calico/templates/calico-kube-controllers.yaml + +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: calico/kube-controllers:v3.13.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-etcd-secrets.yaml + +--- +# Source: calico/templates/calico-typha.yaml + +--- +# Source: calico/templates/configure-canal.yaml + `))) + +func GenerateCalicoFiles(cfg *kubekeyapi.ClusterCfg) (string, error) { + return util.Render(calicoTempl, util.Data{ + "KubePodsCIDR": cfg.Network.KubePodsCIDR, + }) +} diff --git a/plugins/network/flannel/tmpl.go b/plugins/network/flannel/tmpl.go new file mode 100644 index 00000000..f161a64c --- /dev/null +++ b/plugins/network/flannel/tmpl.go @@ -0,0 +1,620 @@ +package flannel + +import ( + "github.com/lithammer/dedent" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/util" + "text/template" +) + +var flannelTempl = template.Must(template.New("kubeadmCfg").Parse( + dedent.Dedent(`--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.flannel.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-flannel" + - pathPrefix: "/run/flannel" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unused in CaaSP + rule: 'RunAsAny' +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['psp.flannel.unprivileged'] + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: + - kind: ServiceAccount + name: flannel + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "{{ .KubePodsCIDR }}", + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-amd64 + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.11.0-amd64 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.11.0-amd64 + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-arm64 + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - arm64 + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.11.0-arm64 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.11.0-arm64 + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-arm + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - arm + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.11.0-arm + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.11.0-arm + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-ppc64le + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - ppc64le + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.11.0-ppc64le + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.11.0-ppc64le + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds-s390x + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - s390x + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni + image: quay.io/coreos/flannel:v0.11.0-s390x + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + image: quay.io/coreos/flannel:v0.11.0-s390x + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + + `))) + +func GenerateFlannelFiles(cfg *kubekeyapi.ClusterCfg) (string, error) { + return util.Render(flannelTempl, util.Data{ + "KubePodsCIDR": cfg.Network.KubePodsCIDR, + }) +} diff --git a/plugins/network/network.go b/plugins/network/network.go new file mode 100644 index 00000000..0a7c3d54 --- /dev/null +++ b/plugins/network/network.go @@ -0,0 +1,77 @@ +package network + +import ( + "encoding/base64" + "fmt" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/plugins/network/calico" + "github.com/pixiake/kubekey/plugins/network/flannel" + "github.com/pixiake/kubekey/util/manager" + "github.com/pixiake/kubekey/util/ssh" + "github.com/pkg/errors" +) + +func DeployNetworkPlugin(mgr *manager.Manager) error { + mgr.Logger.Infoln("Generate etcd certs") + + return mgr.RunTaskOnMasterNodes(deployNetworkPlugin, true) +} + +func deployNetworkPlugin(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + if mgr.Runner.Index == 0 { + switch mgr.Cluster.Network.Plugin { + case "calico": + err := deployCalico(mgr) + return err + case "flannel": + err := deployFlannel(mgr) + return err + case "macvlan": + err := deployMacvlan(mgr) + return err + default: + return errors.New(fmt.Sprintf("This network plugin is not supported: %s", mgr.Cluster.Network.Plugin)) + } + } + return nil +} + +func deployCalico(mgr *manager.Manager) error { + calicoFile, err := calico.GenerateCalicoFiles(mgr.Cluster) + if err != nil { + return err + } + calicoFileBase64 := base64.StdEncoding.EncodeToString([]byte(calicoFile)) + _, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/calico.yaml\"", calicoFileBase64)) + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "failed to generate calico file") + } + + _, err2 := mgr.Runner.RunCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/calico.yaml") + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to deploy calico") + } + return nil +} + +func deployFlannel(mgr *manager.Manager) error { + flannelFile, err := flannel.GenerateFlannelFiles(mgr.Cluster) + if err != nil { + return err + } + flannelFileBase64 := base64.StdEncoding.EncodeToString([]byte(flannelFile)) + _, err1 := mgr.Runner.RunCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /etc/kubernetes/flannel.yaml\"", flannelFileBase64)) + if err1 != nil { + return errors.Wrap(errors.WithStack(err1), "failed to generate flannel file") + } + + _, err2 := mgr.Runner.RunCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/flannel.yaml") + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to deploy flannel") + } + return nil +} + +func deployMacvlan(mgr *manager.Manager) error { + return nil +} diff --git a/util/runner/runner.go b/util/runner/runner.go index fd0676dd..c45d28db 100644 --- a/util/runner/runner.go +++ b/util/runner/runner.go @@ -33,11 +33,11 @@ func (r *Runner) RunCmd(cmd string) (string, error) { } if err != nil { - return "", err + return output, err } if output != "" { - if strings.Contains(cmd, "base64") && strings.Contains(cmd, "--wrap=0") { + if strings.Contains(cmd, "base64") && strings.Contains(cmd, "--wrap=0") || strings.Contains(cmd, "make-ssl-etcd.sh") || strings.Contains(cmd, "docker-install.sh") { } else { fmt.Printf("[%s %s] MSG:\n", r.Host.HostName, r.Host.SSHAddress) fmt.Println(output)