Merge pull request #1962 from pixiake/master

Added new network plugin Hybridnet
This commit is contained in:
KubeSphere CI Bot 2023-08-24 09:11:00 +08:00 committed by GitHub
commit c7aec0faa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 382 additions and 16 deletions

View File

@ -51,6 +51,7 @@ const (
DefaultFlannelCniPluginVersion = "v1.1.2"
DefaultCniVersion = "v1.2.0"
DefaultCiliumVersion = "v1.11.7"
DefaulthybridnetVersion = "v0.8.6"
DefaultKubeovnVersion = "v1.10.6"
DefalutMultusVersion = "v3.8"
DefaultHelmVersion = "v3.9.0"

View File

@ -17,13 +17,14 @@
package v1alpha2
type NetworkConfig struct {
Plugin string `yaml:"plugin" json:"plugin,omitempty"`
KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"`
KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"`
Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"`
Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"`
Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"`
MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"`
Plugin string `yaml:"plugin" json:"plugin,omitempty"`
KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"`
KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"`
Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"`
Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"`
Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"`
MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"`
Hybridnet HybridnetCfg `yaml:"hybridnet" json:"hybridnet,omitempty"`
}
type CalicoCfg struct {
@ -90,6 +91,36 @@ type KubeOvnPinger struct {
PingerExternalDomain string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"`
}
type HybridnetCfg struct {
DefaultNetworkType string `yaml:"defaultNetworkType" json:"defaultNetworkType,omitempty"`
EnableNetworkPolicy *bool `yaml:"enableNetworkPolicy" json:"enableNetworkPolicy,omitempty"`
Init *bool `yaml:"init" json:"init,omitempty"`
PreferVxlanInterfaces string `yaml:"preferVxlanInterfaces" json:"preferVxlanInterfaces,omitempty"`
PreferVlanInterfaces string `yaml:"preferVlanInterfaces" json:"preferVlanInterfaces,omitempty"`
PreferBGPInterfaces string `yaml:"preferBGPInterfaces" json:"preferBGPInterfaces,omitempty"`
Networks []HybridnetNetwork `yaml:"networks" json:"networks,omitempty"`
}
type HybridnetNetwork struct {
Name string `yaml:"name" json:"name,omitempty"`
NetID *int `yaml:"netID" json:"netID,omitempty"`
Type string `yaml:"type" json:"type,omitempty"`
Mode string `yaml:"mode" json:"mode,omitempty"`
NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
Subnets []HybridnetSubnet `yaml:"subnets" json:"subnets,omitempty"`
}
type HybridnetSubnet struct {
Name string `yaml:"name" json:"name,omitempty"`
NetID *int `yaml:"netID" json:"netID,omitempty"`
CIDR string `yaml:"cidr" json:"cidr,omitempty"`
Gateway string `yaml:"gateway" json:"gateway,omitempty"`
Start string `yaml:"start" json:"start,omitempty"`
End string `yaml:"end" json:"end,omitempty"`
ReservedIPs []string `yaml:"reservedIPs" json:"reservedIPs,omitempty"`
ExcludeIPs []string `yaml:"excludeIPs" json:"excludeIPs,omitempty"`
}
func (k *KubeovnCfg) KubeovnCheckGateway() bool {
if k.KubeOvnController.CheckGateway == nil {
return true
@ -151,3 +182,19 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
}
return *c.DefaultIPPOOL
}
// EnableInit is used to determine whether to create default network
func (h *HybridnetCfg) EnableInit() bool {
if h.Init == nil {
return true
}
return *h.Init
}
// NetworkPolicy is used to determine whether to enable network policy
func (h *HybridnetCfg) NetworkPolicy() bool {
if h.EnableNetworkPolicy == nil {
return true
}
return *h.EnableNetworkPolicy
}

View File

@ -55,10 +55,11 @@ const (
IPv4Regexp = "[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+"
IPv6Regexp = "[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}"
Calico = "calico"
Flannel = "flannel"
Cilium = "cilium"
Kubeovn = "kubeovn"
Calico = "calico"
Flannel = "flannel"
Cilium = "cilium"
Kubeovn = "kubeovn"
Hybridnet = "hybridnet"
Docker = "docker"
Crictl = "crictl"

View File

@ -54,6 +54,35 @@ func (image Image) ImageName() string {
return fmt.Sprintf("%s:%s", image.ImageRepo(), image.Tag)
}
// ImageNamespace is used to get image's namespace
func (image Image) ImageNamespace() string {
if os.Getenv("KKZONE") == "cn" {
if image.RepoAddr == "" || image.RepoAddr == cnRegistry {
image.NamespaceOverride = cnNamespaceOverride
}
}
if image.NamespaceOverride != "" {
return image.NamespaceOverride
} else {
return image.Namespace
}
}
// ImageRegistryAddr is used to get image's registry address.
func (image Image) ImageRegistryAddr() string {
if os.Getenv("KKZONE") == "cn" {
if image.RepoAddr == "" || image.RepoAddr == cnRegistry {
image.RepoAddr = cnRegistry
}
}
if image.RepoAddr != "" {
return image.RepoAddr
} else {
return "docker.io"
}
}
// ImageRepo is used to generate image's repo address.
func (image Image) ImageRepo() string {
var prefix string

View File

@ -126,6 +126,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
"flannel-cni-plugin": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel-cni-plugin", Tag: kubekeyv1alpha2.DefaultFlannelCniPluginVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")},
"cilium": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")},
"cilium-operator-generic": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")},
"hybridnet": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "hybridnetdev", Repo: "hybridnet", Tag: kubekeyv1alpha2.DefaulthybridnetVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "hybridnet")},
"kubeovn": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyv1alpha2.DefaultKubeovnVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "kubeovn")},
"multus": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "multus-cni", Tag: kubekeyv1alpha2.DefalutMultusVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.Contains(kubeConf.Cluster.Network.Plugin, "multus")},
// storage

Binary file not shown.

View File

@ -48,6 +48,8 @@ func (d *DeployNetworkPluginModule) Init() {
d.Tasks = deployCilium(d)
case common.Kubeovn:
d.Tasks = deployKubeOVN(d)
case common.Hybridnet:
d.Tasks = deployHybridnet(d)
default:
return
}
@ -239,8 +241,8 @@ func deployCilium(d *DeployNetworkPluginModule) []task.Interface {
}
syncCiliumChart := &task.RemoteTask{
Name: "SyncKubeBinary",
Desc: "Synchronize kubernetes binaries",
Name: "SyncCiliumChart",
Desc: "Synchronize cilium chart",
Hosts: d.Runtime.GetHostsByRole(common.Master),
Prepare: new(common.OnlyFirstMaster),
Action: new(SyncCiliumChart),
@ -335,6 +337,41 @@ func deployKubeOVN(d *DeployNetworkPluginModule) []task.Interface {
}
}
func deployHybridnet(d *DeployNetworkPluginModule) []task.Interface {
releaseHybridnetChart := &task.LocalTask{
Name: "GenerateHybridnetChart",
Desc: "Generate hybridnet chart",
Action: new(ReleaseHybridnetChart),
}
syncHybridnetChart := &task.RemoteTask{
Name: "SyncHybridnetChart",
Desc: "Synchronize hybridnet chart",
Hosts: d.Runtime.GetHostsByRole(common.Master),
Prepare: new(common.OnlyFirstMaster),
Action: new(SyncHybridnetChart),
Parallel: true,
Retry: 2,
}
deploy := &task.RemoteTask{
Name: "DeployHybridnet",
Desc: "Deploy hybridnet",
Hosts: d.Runtime.GetHostsByRole(common.Master),
Prepare: new(common.OnlyFirstMaster),
Action: new(DeployHybridnet),
Parallel: true,
Retry: 5,
}
return []task.Interface{
releaseHybridnetChart,
syncHybridnetChart,
deploy,
}
}
func K8sVersionAtLeast(version string, compare string) bool {
cmp, err := versionutil.MustParseSemantic(version).Compare(compare)
if err != nil {

View File

@ -19,11 +19,12 @@ package network
import (
"embed"
"fmt"
"github.com/pkg/errors"
"io"
"os"
"path/filepath"
"github.com/pkg/errors"
"strings"
"time"
"github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
@ -34,7 +35,7 @@ import (
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates"
)
//go:embed cilium-1.11.7.tgz
//go:embed cilium-1.11.7.tgz hybridnet-0.6.6.tgz
var f embed.FS
@ -306,3 +307,117 @@ func (c *ChmodKubectlKo) Execute(runtime connector.Runtime) error {
}
return nil
}
// ReleaseHybridnetChart is used to release hybridnet chart to local path
type ReleaseHybridnetChart struct {
common.KubeAction
}
func (r *ReleaseHybridnetChart) Execute(runtime connector.Runtime) error {
fs, err := os.Create(fmt.Sprintf("%s/hybridnet.tgz", runtime.GetWorkDir()))
if err != nil {
return err
}
chartFile, err := f.Open("hybridnet-0.6.6.tgz")
if err != nil {
return err
}
defer chartFile.Close()
_, err = io.Copy(fs, chartFile)
if err != nil {
return err
}
return nil
}
// SyncHybridnetChart is used to sync hybridnet chart to contronplane
type SyncHybridnetChart struct {
common.KubeAction
}
func (s *SyncHybridnetChart) Execute(runtime connector.Runtime) error {
src := filepath.Join(runtime.GetWorkDir(), "hybridnet.tgz")
dst := filepath.Join(common.TmpDir, "hybridnet.tgz")
if err := runtime.GetRunner().Scp(src, dst); err != nil {
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync hybridnet chart failed"))
}
if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/hybridnet.tgz /etc/kubernetes", common.TmpDir), true); err != nil {
return errors.Wrap(errors.WithStack(err), "sync hybrident chart failed")
}
return nil
}
type DeployHybridnet struct {
common.KubeAction
}
func (d *DeployHybridnet) Execute(runtime connector.Runtime) error {
cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install hybridnet /etc/kubernetes/hybridnet.tgz --namespace kube-system "+
"--set images.hybridnet.image=%s/%s "+
"--set images.hybridnet.tag=%s "+
"--set images.registryURL=%s ",
images.GetImage(runtime, d.KubeConf, "hybridnet").ImageNamespace(),
images.GetImage(runtime, d.KubeConf, "hybridnet").Repo,
images.GetImage(runtime, d.KubeConf, "hybridnet").Tag,
images.GetImage(runtime, d.KubeConf, "hybridnet").ImageRegistryAddr(),
)
if d.KubeConf.Cluster.Network.Hybridnet.EnableInit() {
cmd = fmt.Sprintf("%s --set init.cidr=%s", cmd, d.KubeConf.Cluster.Network.KubePodsCIDR)
} else {
cmd = fmt.Sprintf("%s --set init=null", cmd)
}
if !d.KubeConf.Cluster.Network.Hybridnet.NetworkPolicy() {
cmd = fmt.Sprintf("%s --set daemon.enableNetworkPolicy=false", cmd)
}
if d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces != "" {
cmd = fmt.Sprintf("%s --set daemon.preferBGPInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces)
}
if d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces != "" {
cmd = fmt.Sprintf("%s --set daemon.preferVlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces)
}
if d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces != "" {
cmd = fmt.Sprintf("%s --set daemon.preferVxlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces)
}
if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil {
return errors.Wrap(errors.WithStack(err), "deploy hybridnet failed")
}
if len(d.KubeConf.Cluster.Network.Hybridnet.Networks) != 0 {
templateAction := action.Template{
Template: templates.HybridnetNetworks,
Dst: filepath.Join(common.KubeConfigDir, templates.HybridnetNetworks.Name()),
Data: util.Data{
"Networks": d.KubeConf.Cluster.Network.Hybridnet.Networks,
},
}
templateAction.Init(nil, nil)
if err := templateAction.Execute(runtime); err != nil {
return err
}
for i := 0; i < 30; i++ {
fmt.Println("Waiting for hybridnet webhook running ... ", i+1)
time.Sleep(10 * time.Second)
output, _ := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get pod -n kube-system -l app=hybridnet,component=webhook | grep Running", false)
if strings.Contains(output, "1/1") {
time.Sleep(50 * time.Second)
break
}
}
if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/hybridnet-networks.yaml", true); err != nil {
return errors.Wrap(errors.WithStack(err), "apply hybridnet networks failed")
}
}
return nil
}

View File

@ -0,0 +1,79 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package templates
import (
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
"github.com/lithammer/dedent"
"text/template"
)
var HybridnetNetworks = template.Must(template.New("hybridnet-networks.yaml").Funcs(utils.FuncMap).Parse(
dedent.Dedent(`
{{- range $index, $network := .Networks }}
---
apiVersion: networking.alibaba.com/v1
kind: Network
metadata:
name: {{ $network.Name }}
spec:
{{- if $network.NetID }}
netID: {{ $network.NetID }}
{{- end }}
type: {{ $network.Type }}
{{- if $network.Mode }}
mode: {{ $network.Mode }}
{{- end }}
{{- if $network.NodeSelector }}
nodeSelector:
{{ toYaml $network.NodeSelector | indent 4 }}
{{- end }}
{{- range $network.Subnets }}
---
apiVersion: networking.alibaba.com/v1
kind: Subnet
metadata:
name: {{ .Name }}
spec:
network: {{ $network.Name }}
{{- if .NetID }}
netID: {{ .NetID }}
{{- end }}
range:
version: "4"
cidr: "{{ .CIDR }}"
{{- if .Gateway }}
gateway: "{{ .Gateway }}"
{{- end }}
{{- if .Start}}
start: "{{ .Start }}"
{{- end}}
{{- if .End}}
end: "{{ .End }}"
{{- end }}
{{- if .ReservedIPs }}
reservedIPs:
{{ toYaml .ReservedIPs | indent 4 }}
{{- end }}
{{- if .ExcludeIPs }}
excludeIPs:
{{ toYaml .ReservedIPs | indent 4 }}
{{- end }}
{{- end }}
{{- end }}
`)))

View File

@ -1,3 +1,4 @@
# Cluster Configuration Sample
```yaml
apiVersion: kubekey.kubesphere.io/v1alpha2
kind: Cluster
@ -230,3 +231,58 @@ spec:
# - name substring website.tld website.namespace.svc.cluster.local
```
# Network Configuration sample
## Hybridnet
To learn more about hybridnet, check out https://github.com/alibaba/hybridnet
```yaml
network:
plugin: hybridnet
hybridnet:
defaultNetworkType: Overlay
enableNetworkPolicy: false
init: false
preferVxlanInterfaces: eth0
preferVlanInterfaces: eth0
preferBGPInterfaces: eth0
networks:
- name: "net1"
type: Underlay
nodeSelector:
network: "net1"
subnets:
- name: "subnet-10"
netID: 10
cidr: "192.168.10.0/24"
gateway: "192.168.10.1"
- name: "subnet-11"
netID: 11
cidr: "192.168.11.0/24"
gateway: "192.168.11.1"
- name: "net2"
type: Underlay
nodeSelector:
network: "net2"
subnets:
- name: "subnet-30"
netID: 30
cidr: "192.168.30.0/24"
gateway: "192.168.30.1"
- name: "subnet-31"
netID: 31
cidr: "192.168.31.0/24"
gateway: "192.168.31.1"
- name: "net3"
type: Underlay
netID: 0
nodeSelector:
network: "net3"
subnets:
- name: "subnet-50"
cidr: "192.168.50.0/24"
gateway: "192.168.50.1"
start: "192.168.50.100"
end: "192.168.50.200"
reservedIPs: ["192.168.50.101","192.168.50.102"]
excludeIPs: ["192.168.50.111","192.168.50.112"]
```