diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go index 9fc94d08..60232e01 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ b/cmd/kk/apis/kubekey/v1alpha2/default.go @@ -51,6 +51,7 @@ const ( DefaultFlannelCniPluginVersion = "v1.1.2" DefaultCniVersion = "v1.2.0" DefaultCiliumVersion = "v1.11.7" + DefaulthybridnetVersion = "v0.8.6" DefaultKubeovnVersion = "v1.10.6" DefalutMultusVersion = "v3.8" DefaultHelmVersion = "v3.9.0" diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go index 3f1759da..0fe7f8ab 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/network_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/network_types.go @@ -17,13 +17,14 @@ package v1alpha2 type NetworkConfig struct { - Plugin string `yaml:"plugin" json:"plugin,omitempty"` - KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"` - KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"` - Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"` - Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"` - Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"` - MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"` + Plugin string `yaml:"plugin" json:"plugin,omitempty"` + KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"` + KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"` + Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"` + Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"` + Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"` + MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"` + Hybridnet HybridnetCfg `yaml:"hybridnet" json:"hybridnet,omitempty"` } type CalicoCfg struct { @@ -90,6 +91,36 @@ type KubeOvnPinger struct { PingerExternalDomain string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"` } +type HybridnetCfg struct { + DefaultNetworkType string `yaml:"defaultNetworkType" json:"defaultNetworkType,omitempty"` + EnableNetworkPolicy *bool `yaml:"enableNetworkPolicy" json:"enableNetworkPolicy,omitempty"` + Init *bool `yaml:"init" json:"init,omitempty"` + PreferVxlanInterfaces string `yaml:"preferVxlanInterfaces" json:"preferVxlanInterfaces,omitempty"` + PreferVlanInterfaces string `yaml:"preferVlanInterfaces" json:"preferVlanInterfaces,omitempty"` + PreferBGPInterfaces string `yaml:"preferBGPInterfaces" json:"preferBGPInterfaces,omitempty"` + Networks []HybridnetNetwork `yaml:"networks" json:"networks,omitempty"` +} + +type HybridnetNetwork struct { + Name string `yaml:"name" json:"name,omitempty"` + NetID *int `yaml:"netID" json:"netID,omitempty"` + Type string `yaml:"type" json:"type,omitempty"` + Mode string `yaml:"mode" json:"mode,omitempty"` + NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` + Subnets []HybridnetSubnet `yaml:"subnets" json:"subnets,omitempty"` +} + +type HybridnetSubnet struct { + Name string `yaml:"name" json:"name,omitempty"` + NetID *int `yaml:"netID" json:"netID,omitempty"` + CIDR string `yaml:"cidr" json:"cidr,omitempty"` + Gateway string `yaml:"gateway" json:"gateway,omitempty"` + Start string `yaml:"start" json:"start,omitempty"` + End string `yaml:"end" json:"end,omitempty"` + ReservedIPs []string `yaml:"reservedIPs" json:"reservedIPs,omitempty"` + ExcludeIPs []string `yaml:"excludeIPs" json:"excludeIPs,omitempty"` +} + func (k *KubeovnCfg) KubeovnCheckGateway() bool { if k.KubeOvnController.CheckGateway == nil { return true @@ -151,3 +182,19 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool { } return *c.DefaultIPPOOL } + +// EnableInit is used to determine whether to create default network +func (h *HybridnetCfg) EnableInit() bool { + if h.Init == nil { + return true + } + return *h.Init +} + +// NetworkPolicy is used to determine whether to enable network policy +func (h *HybridnetCfg) NetworkPolicy() bool { + if h.EnableNetworkPolicy == nil { + return true + } + return *h.EnableNetworkPolicy +} diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go index f883b2da..4b24d7a3 100644 --- a/cmd/kk/pkg/common/common.go +++ b/cmd/kk/pkg/common/common.go @@ -55,10 +55,11 @@ const ( IPv4Regexp = "[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+" IPv6Regexp = "[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}" - Calico = "calico" - Flannel = "flannel" - Cilium = "cilium" - Kubeovn = "kubeovn" + Calico = "calico" + Flannel = "flannel" + Cilium = "cilium" + Kubeovn = "kubeovn" + Hybridnet = "hybridnet" Docker = "docker" Crictl = "crictl" diff --git a/cmd/kk/pkg/images/images.go b/cmd/kk/pkg/images/images.go index 48a7035b..f314cbea 100644 --- a/cmd/kk/pkg/images/images.go +++ b/cmd/kk/pkg/images/images.go @@ -54,6 +54,35 @@ func (image Image) ImageName() string { return fmt.Sprintf("%s:%s", image.ImageRepo(), image.Tag) } +// ImageNamespace is used to get image's namespace +func (image Image) ImageNamespace() string { + if os.Getenv("KKZONE") == "cn" { + if image.RepoAddr == "" || image.RepoAddr == cnRegistry { + image.NamespaceOverride = cnNamespaceOverride + } + } + + if image.NamespaceOverride != "" { + return image.NamespaceOverride + } else { + return image.Namespace + } +} + +// ImageRegistryAddr is used to get image's registry address. +func (image Image) ImageRegistryAddr() string { + if os.Getenv("KKZONE") == "cn" { + if image.RepoAddr == "" || image.RepoAddr == cnRegistry { + image.RepoAddr = cnRegistry + } + } + if image.RepoAddr != "" { + return image.RepoAddr + } else { + return "docker.io" + } +} + // ImageRepo is used to generate image's repo address. func (image Image) ImageRepo() string { var prefix string diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go index e77e141b..f8d59f8d 100644 --- a/cmd/kk/pkg/images/tasks.go +++ b/cmd/kk/pkg/images/tasks.go @@ -126,6 +126,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s "flannel-cni-plugin": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel-cni-plugin", Tag: kubekeyv1alpha2.DefaultFlannelCniPluginVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")}, "cilium": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, "cilium-operator-generic": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, + "hybridnet": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "hybridnetdev", Repo: "hybridnet", Tag: kubekeyv1alpha2.DefaulthybridnetVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "hybridnet")}, "kubeovn": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyv1alpha2.DefaultKubeovnVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "kubeovn")}, "multus": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "multus-cni", Tag: kubekeyv1alpha2.DefalutMultusVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.Contains(kubeConf.Cluster.Network.Plugin, "multus")}, // storage diff --git a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz b/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz new file mode 100644 index 00000000..a85ac275 Binary files /dev/null and b/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz differ diff --git a/cmd/kk/pkg/plugins/network/modules.go b/cmd/kk/pkg/plugins/network/modules.go index 5ebb14f4..21fb5e31 100644 --- a/cmd/kk/pkg/plugins/network/modules.go +++ b/cmd/kk/pkg/plugins/network/modules.go @@ -48,6 +48,8 @@ func (d *DeployNetworkPluginModule) Init() { d.Tasks = deployCilium(d) case common.Kubeovn: d.Tasks = deployKubeOVN(d) + case common.Hybridnet: + d.Tasks = deployHybridnet(d) default: return } @@ -239,8 +241,8 @@ func deployCilium(d *DeployNetworkPluginModule) []task.Interface { } syncCiliumChart := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", + Name: "SyncCiliumChart", + Desc: "Synchronize cilium chart", Hosts: d.Runtime.GetHostsByRole(common.Master), Prepare: new(common.OnlyFirstMaster), Action: new(SyncCiliumChart), @@ -335,6 +337,41 @@ func deployKubeOVN(d *DeployNetworkPluginModule) []task.Interface { } } +func deployHybridnet(d *DeployNetworkPluginModule) []task.Interface { + + releaseHybridnetChart := &task.LocalTask{ + Name: "GenerateHybridnetChart", + Desc: "Generate hybridnet chart", + Action: new(ReleaseHybridnetChart), + } + + syncHybridnetChart := &task.RemoteTask{ + Name: "SyncHybridnetChart", + Desc: "Synchronize hybridnet chart", + Hosts: d.Runtime.GetHostsByRole(common.Master), + Prepare: new(common.OnlyFirstMaster), + Action: new(SyncHybridnetChart), + Parallel: true, + Retry: 2, + } + + deploy := &task.RemoteTask{ + Name: "DeployHybridnet", + Desc: "Deploy hybridnet", + Hosts: d.Runtime.GetHostsByRole(common.Master), + Prepare: new(common.OnlyFirstMaster), + Action: new(DeployHybridnet), + Parallel: true, + Retry: 5, + } + + return []task.Interface{ + releaseHybridnetChart, + syncHybridnetChart, + deploy, + } +} + func K8sVersionAtLeast(version string, compare string) bool { cmp, err := versionutil.MustParseSemantic(version).Compare(compare) if err != nil { diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go index 57d4b6c3..68824b39 100644 --- a/cmd/kk/pkg/plugins/network/tasks.go +++ b/cmd/kk/pkg/plugins/network/tasks.go @@ -19,11 +19,12 @@ package network import ( "embed" "fmt" + "github.com/pkg/errors" "io" "os" "path/filepath" - - "github.com/pkg/errors" + "strings" + "time" "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" @@ -34,7 +35,7 @@ import ( "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates" ) -//go:embed cilium-1.11.7.tgz +//go:embed cilium-1.11.7.tgz hybridnet-0.6.6.tgz var f embed.FS @@ -306,3 +307,117 @@ func (c *ChmodKubectlKo) Execute(runtime connector.Runtime) error { } return nil } + +// ReleaseHybridnetChart is used to release hybridnet chart to local path +type ReleaseHybridnetChart struct { + common.KubeAction +} + +func (r *ReleaseHybridnetChart) Execute(runtime connector.Runtime) error { + fs, err := os.Create(fmt.Sprintf("%s/hybridnet.tgz", runtime.GetWorkDir())) + if err != nil { + return err + } + chartFile, err := f.Open("hybridnet-0.6.6.tgz") + if err != nil { + return err + } + defer chartFile.Close() + + _, err = io.Copy(fs, chartFile) + if err != nil { + return err + } + + return nil +} + +// SyncHybridnetChart is used to sync hybridnet chart to contronplane +type SyncHybridnetChart struct { + common.KubeAction +} + +func (s *SyncHybridnetChart) Execute(runtime connector.Runtime) error { + src := filepath.Join(runtime.GetWorkDir(), "hybridnet.tgz") + dst := filepath.Join(common.TmpDir, "hybridnet.tgz") + if err := runtime.GetRunner().Scp(src, dst); err != nil { + return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync hybridnet chart failed")) + } + if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/hybridnet.tgz /etc/kubernetes", common.TmpDir), true); err != nil { + return errors.Wrap(errors.WithStack(err), "sync hybrident chart failed") + } + return nil +} + +type DeployHybridnet struct { + common.KubeAction +} + +func (d *DeployHybridnet) Execute(runtime connector.Runtime) error { + + cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install hybridnet /etc/kubernetes/hybridnet.tgz --namespace kube-system "+ + "--set images.hybridnet.image=%s/%s "+ + "--set images.hybridnet.tag=%s "+ + "--set images.registryURL=%s ", + images.GetImage(runtime, d.KubeConf, "hybridnet").ImageNamespace(), + images.GetImage(runtime, d.KubeConf, "hybridnet").Repo, + images.GetImage(runtime, d.KubeConf, "hybridnet").Tag, + images.GetImage(runtime, d.KubeConf, "hybridnet").ImageRegistryAddr(), + ) + + if d.KubeConf.Cluster.Network.Hybridnet.EnableInit() { + cmd = fmt.Sprintf("%s --set init.cidr=%s", cmd, d.KubeConf.Cluster.Network.KubePodsCIDR) + } else { + cmd = fmt.Sprintf("%s --set init=null", cmd) + } + + if !d.KubeConf.Cluster.Network.Hybridnet.NetworkPolicy() { + cmd = fmt.Sprintf("%s --set daemon.enableNetworkPolicy=false", cmd) + } + + if d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces != "" { + cmd = fmt.Sprintf("%s --set daemon.preferBGPInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces) + } + + if d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces != "" { + cmd = fmt.Sprintf("%s --set daemon.preferVlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces) + } + + if d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces != "" { + cmd = fmt.Sprintf("%s --set daemon.preferVxlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces) + } + + if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil { + return errors.Wrap(errors.WithStack(err), "deploy hybridnet failed") + } + + if len(d.KubeConf.Cluster.Network.Hybridnet.Networks) != 0 { + templateAction := action.Template{ + Template: templates.HybridnetNetworks, + Dst: filepath.Join(common.KubeConfigDir, templates.HybridnetNetworks.Name()), + Data: util.Data{ + "Networks": d.KubeConf.Cluster.Network.Hybridnet.Networks, + }, + } + + templateAction.Init(nil, nil) + if err := templateAction.Execute(runtime); err != nil { + return err + } + + for i := 0; i < 30; i++ { + fmt.Println("Waiting for hybridnet webhook running ... ", i+1) + time.Sleep(10 * time.Second) + output, _ := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get pod -n kube-system -l app=hybridnet,component=webhook | grep Running", false) + if strings.Contains(output, "1/1") { + time.Sleep(50 * time.Second) + break + } + } + + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/hybridnet-networks.yaml", true); err != nil { + return errors.Wrap(errors.WithStack(err), "apply hybridnet networks failed") + } + } + return nil +} diff --git a/cmd/kk/pkg/plugins/network/templates/hybridnet.go b/cmd/kk/pkg/plugins/network/templates/hybridnet.go new file mode 100644 index 00000000..03b3b103 --- /dev/null +++ b/cmd/kk/pkg/plugins/network/templates/hybridnet.go @@ -0,0 +1,79 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package templates + +import ( + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" + "github.com/lithammer/dedent" + "text/template" +) + +var HybridnetNetworks = template.Must(template.New("hybridnet-networks.yaml").Funcs(utils.FuncMap).Parse( + dedent.Dedent(` +{{- range $index, $network := .Networks }} +--- +apiVersion: networking.alibaba.com/v1 +kind: Network +metadata: + name: {{ $network.Name }} +spec: +{{- if $network.NetID }} + netID: {{ $network.NetID }} +{{- end }} + type: {{ $network.Type }} +{{- if $network.Mode }} + mode: {{ $network.Mode }} +{{- end }} +{{- if $network.NodeSelector }} + nodeSelector: +{{ toYaml $network.NodeSelector | indent 4 }} +{{- end }} + +{{- range $network.Subnets }} +--- +apiVersion: networking.alibaba.com/v1 +kind: Subnet +metadata: + name: {{ .Name }} +spec: + network: {{ $network.Name }} +{{- if .NetID }} + netID: {{ .NetID }} +{{- end }} + range: + version: "4" + cidr: "{{ .CIDR }}" +{{- if .Gateway }} + gateway: "{{ .Gateway }}" +{{- end }} +{{- if .Start}} + start: "{{ .Start }}" +{{- end}} +{{- if .End}} + end: "{{ .End }}" +{{- end }} +{{- if .ReservedIPs }} + reservedIPs: +{{ toYaml .ReservedIPs | indent 4 }} +{{- end }} +{{- if .ExcludeIPs }} + excludeIPs: +{{ toYaml .ReservedIPs | indent 4 }} +{{- end }} +{{- end }} +{{- end }} + `))) diff --git a/docs/config-example.md b/docs/config-example.md index a842760f..3a8796bb 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -1,3 +1,4 @@ +# Cluster Configuration Sample ```yaml apiVersion: kubekey.kubesphere.io/v1alpha2 kind: Cluster @@ -230,3 +231,58 @@ spec: # - name substring website.tld website.namespace.svc.cluster.local ``` + +# Network Configuration sample +## Hybridnet +To learn more about hybridnet, check out https://github.com/alibaba/hybridnet +```yaml + network: + plugin: hybridnet + hybridnet: + defaultNetworkType: Overlay + enableNetworkPolicy: false + init: false + preferVxlanInterfaces: eth0 + preferVlanInterfaces: eth0 + preferBGPInterfaces: eth0 + networks: + - name: "net1" + type: Underlay + nodeSelector: + network: "net1" + subnets: + - name: "subnet-10" + netID: 10 + cidr: "192.168.10.0/24" + gateway: "192.168.10.1" + - name: "subnet-11" + netID: 11 + cidr: "192.168.11.0/24" + gateway: "192.168.11.1" + - name: "net2" + type: Underlay + nodeSelector: + network: "net2" + subnets: + - name: "subnet-30" + netID: 30 + cidr: "192.168.30.0/24" + gateway: "192.168.30.1" + - name: "subnet-31" + netID: 31 + cidr: "192.168.31.0/24" + gateway: "192.168.31.1" + - name: "net3" + type: Underlay + netID: 0 + nodeSelector: + network: "net3" + subnets: + - name: "subnet-50" + cidr: "192.168.50.0/24" + gateway: "192.168.50.1" + start: "192.168.50.100" + end: "192.168.50.200" + reservedIPs: ["192.168.50.101","192.168.50.102"] + excludeIPs: ["192.168.50.111","192.168.50.112"] +```