From 26f37bb432847379b7bf9ce22c3a63a4cdc3ef48 Mon Sep 17 00:00:00 2001 From: pixiake Date: Fri, 18 Aug 2023 17:59:03 +0800 Subject: [PATCH] Supports custom configuration of cluster dns Signed-off-by: pixiake --- cmd/kk/apis/kubekey/v1alpha2/cluster_types.go | 1 + cmd/kk/apis/kubekey/v1alpha2/default.go | 1 + cmd/kk/apis/kubekey/v1alpha2/dns_types.go | 41 +++ cmd/kk/cmd/version/version.go | 2 +- cmd/kk/pkg/bootstrap/precheck/tasks.go | 19 +- cmd/kk/pkg/common/loader.go | 8 +- cmd/kk/pkg/images/tasks.go | 2 +- cmd/kk/pkg/kubernetes/module.go | 73 ++++- cmd/kk/pkg/kubernetes/tasks.go | 261 +++++++++--------- .../templates/v1beta2/kubeadm_config.go | 5 +- cmd/kk/pkg/kubesphere/tasks.go | 6 +- cmd/kk/pkg/phase/nodes/modules.go | 9 - cmd/kk/pkg/plugins/dns/module.go | 113 +++++--- cmd/kk/pkg/plugins/dns/tasks.go | 58 +++- cmd/kk/pkg/plugins/dns/templates/coredns.go | 211 ++++++++++++++ .../dns/templates/coredns_configmap.go | 97 +++++++ .../plugins/dns/templates/coredns_service.go | 86 ------ ...odelocaldns_service.go => nodelocaldns.go} | 8 +- .../dns/templates/nodelocaldns_configmap.go | 37 ++- cmd/kk/pkg/utils/util.go | 7 +- docs/config-example.md | 66 +++++ go.mod | 4 +- 22 files changed, 794 insertions(+), 321 deletions(-) create mode 100644 cmd/kk/apis/kubekey/v1alpha2/dns_types.go create mode 100644 cmd/kk/pkg/plugins/dns/templates/coredns.go create mode 100644 cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go delete mode 100644 cmd/kk/pkg/plugins/dns/templates/coredns_service.go rename cmd/kk/pkg/plugins/dns/templates/{nodelocaldns_service.go => nodelocaldns.go} (95%) diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go index 4dc1cf37..b3b3a2c9 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go @@ -38,6 +38,7 @@ type ClusterSpec struct { ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"` System System `yaml:"system" json:"system,omitempty"` Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` + DNS DNS `yaml:"dns" json:"dns,omitempty"` Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"` Network NetworkConfig `yaml:"network" json:"network,omitempty"` Storage StorageConfig `yaml:"storage" json:"storage,omitempty"` diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go index 7d296c5e..194d456f 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ b/cmd/kk/apis/kubekey/v1alpha2/default.go @@ -115,6 +115,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*Kub clusterCfg.Storage = SetDefaultStorageCfg(cfg) clusterCfg.System = cfg.System clusterCfg.Kubernetes = SetDefaultClusterCfg(cfg) + clusterCfg.DNS = cfg.DNS clusterCfg.Registry = cfg.Registry clusterCfg.Addons = cfg.Addons clusterCfg.KubeSphere = cfg.KubeSphere diff --git a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go new file mode 100644 index 00000000..aeaf0081 --- /dev/null +++ b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go @@ -0,0 +1,41 @@ +/* + Copyright 2023 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha2 + +type DNS struct { + DNSEtcHosts string `yaml:"dnsEtcHosts" json:"dnsEtcHosts"` + CoreDNS CoreDNS `yaml:"coredns" json:"coredns"` + NodeLocalDNS NodeLocalDNS `yaml:"nodelocaldns" json:"nodelocaldns"` +} + +type CoreDNS struct { + AdditionalConfigs string `yaml:"additionalConfigs" json:"additionalConfigs"` + ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` + RewriteBlock string `yaml:"rewriteBlock" json:"rewriteBlock"` + UpstreamDNSServers []string `yaml:"upstreamDNSServers" json:"upstreamDNSServers"` +} + +type NodeLocalDNS struct { + ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` +} + +type ExternalZone struct { + Zones []string `yaml:"zones" json:"zones"` + Nameservers []string `yaml:"nameservers" json:"nameservers"` + Cache int `yaml:"cache" json:"cache"` + Rewrite []string `yaml:"rewrite" json:"rewrite"` +} diff --git a/cmd/kk/cmd/version/version.go b/cmd/kk/cmd/version/version.go index 89c244ab..13b99070 100644 --- a/cmd/kk/cmd/version/version.go +++ b/cmd/kk/cmd/version/version.go @@ -24,7 +24,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" "github.com/kubesphere/kubekey/v3/version" diff --git a/cmd/kk/pkg/bootstrap/precheck/tasks.go b/cmd/kk/pkg/bootstrap/precheck/tasks.go index 17b22704..c6d60cb7 100644 --- a/cmd/kk/pkg/bootstrap/precheck/tasks.go +++ b/cmd/kk/pkg/bootstrap/precheck/tasks.go @@ -168,6 +168,8 @@ func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { nodeK8sVersion = strings.Split(kubeletVersionInfo, " ")[1] host := runtime.RemoteHost() + host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) + if host.IsRole(common.Master) { apiserverVersion, err := runtime.GetRunner().SudoCmd( "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", @@ -175,9 +177,22 @@ func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { if err != nil { return errors.Wrap(err, "get current kube-apiserver version failed") } - nodeK8sVersion = apiserverVersion + + apiserverSemanticVersion, err := versionutil.ParseSemantic(apiserverVersion) + if err != nil { + return errors.Wrap(err, "parse kube-apiserver version failed") + } + + kubeletSemanticVersion, err := versionutil.ParseSemantic(nodeK8sVersion) + if err != nil { + return errors.Wrap(err, "parse kubelet version failed") + } + + if apiserverSemanticVersion.LessThan(kubeletSemanticVersion) { + host.GetCache().Set(common.NodeK8sVersion, apiserverVersion) + } } - host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) + return nil } diff --git a/cmd/kk/pkg/common/loader.go b/cmd/kk/pkg/common/loader.go index 20f9d7ba..06098c03 100644 --- a/cmd/kk/pkg/common/loader.go +++ b/cmd/kk/pkg/common/loader.go @@ -30,7 +30,7 @@ import ( "time" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" k8syaml "k8s.io/apimachinery/pkg/util/yaml" kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" @@ -212,13 +212,13 @@ func (f FileLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { if err := json.Unmarshal(contentToJson, &clusterCfg); err != nil { return nil, errors.Wrap(err, "Failed to unmarshal configuration") } - metadata := result["metadata"].(map[interface{}]interface{}) + metadata := result["metadata"].(map[string]interface{}) objName = metadata["name"].(string) } if result["kind"] == "ConfigMap" || result["kind"] == "ClusterConfiguration" { - metadata := result["metadata"].(map[interface{}]interface{}) - labels := metadata["labels"].(map[interface{}]interface{}) + metadata := result["metadata"].(map[string]interface{}) + labels := metadata["labels"].(map[string]interface{}) clusterCfg.Spec.KubeSphere.Enabled = true v, ok := labels["version"] diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go index afa8651a..e77e141b 100644 --- a/cmd/kk/pkg/images/tasks.go +++ b/cmd/kk/pkg/images/tasks.go @@ -116,7 +116,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s // network "coredns": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "coredns", Repo: "coredns", Tag: corednsTag, Group: kubekeyv1alpha2.K8s, Enable: true}, - "k8s-dns-node-cache": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "k8s-dns-node-cache", Tag: "1.15.12", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodelocaldns()}, + "k8s-dns-node-cache": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "k8s-dns-node-cache", Tag: "1.22.20", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodelocaldns()}, "calico-kube-controllers": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "kube-controllers", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, "calico-cni": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "cni", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, "calico-node": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "node", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, diff --git a/cmd/kk/pkg/kubernetes/module.go b/cmd/kk/pkg/kubernetes/module.go index 4b6c76ae..c948c1c0 100644 --- a/cmd/kk/pkg/kubernetes/module.go +++ b/cmd/kk/pkg/kubernetes/module.go @@ -18,6 +18,8 @@ package kubernetes import ( "fmt" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" "path/filepath" "github.com/pkg/errors" @@ -29,6 +31,7 @@ import ( "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" + dnsTemplates "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" ) type StatusModule struct { @@ -552,6 +555,59 @@ func (p *ProgressiveUpgradeModule) Init() { Parallel: false, } + generateCoreDNS := &task.RemoteTask{ + Name: "GenerateCoreDNS", + Desc: "Generate coredns manifests", + Hosts: p.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + }, + Action: new(dns.GenerateCorednsmanifests), + Parallel: true, + } + + applyCoredns := &task.RemoteTask{ + Name: "DeployCoreDNS", + Desc: "Deploy coredns", + Hosts: p.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + }, + Action: new(dns.DeployCoreDNS), + Parallel: true, + } + + generateNodeLocalDNS := &task.RemoteTask{ + Name: "GenerateNodeLocalDNS", + Desc: "Generate nodelocaldns", + Hosts: p.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + new(dns.EnableNodeLocalDNS), + }, + Action: &action.Template{ + Template: dnsTemplates.NodeLocalDNSService, + Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), + Data: util.Data{ + "NodelocaldnsImage": images.GetImage(p.Runtime, p.KubeConf, "k8s-dns-node-cache").ImageName(), + "DNSEtcHosts": p.KubeConf.Cluster.DNS.DNSEtcHosts, + }, + }, + Parallel: true, + } + + applyNodeLocalDNS := &task.RemoteTask{ + Name: "DeployNodeLocalDNS", + Desc: "Deploy nodelocaldns", + Hosts: p.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + new(dns.EnableNodeLocalDNS)}, + Action: new(dns.DeployNodeLocalDNS), + Parallel: true, + Retry: 5, + } + upgradeKubeWorker := &task.RemoteTask{ Name: "UpgradeClusterOnWorker", Desc: "Upgrade cluster on worker", @@ -564,18 +620,6 @@ func (p *ProgressiveUpgradeModule) Init() { Parallel: false, } - reconfigureDNS := &task.RemoteTask{ - Name: "ReconfigureCoreDNS", - Desc: "Reconfigure CoreDNS", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualPlanVersion), - }, - Action: &ReconfigureDNS{ModuleName: p.Name}, - Parallel: false, - } - currentVersion := &task.LocalTask{ Name: "SetCurrentK8sVersion", Desc: "Set current k8s version", @@ -591,7 +635,10 @@ func (p *ProgressiveUpgradeModule) Init() { upgradeKubeMaster, clusterStatus, upgradeKubeWorker, - reconfigureDNS, + generateCoreDNS, + applyCoredns, + generateNodeLocalDNS, + applyNodeLocalDNS, currentVersion, } } diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go index 1e130a5d..26128da1 100644 --- a/cmd/kk/pkg/kubernetes/tasks.go +++ b/cmd/kk/pkg/kubernetes/tasks.go @@ -39,7 +39,6 @@ import ( "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" @@ -47,8 +46,6 @@ import ( "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates/v1beta2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" - dnsTemplates "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" ) @@ -709,6 +706,7 @@ type UpgradeKubeMaster struct { func (u *UpgradeKubeMaster) Execute(runtime connector.Runtime) error { host := runtime.RemoteHost() + if err := KubeadmUpgradeTasks(runtime, u); err != nil { return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade cluster using kubeadm failed: %s", host.GetName())) } @@ -807,6 +805,7 @@ type KubeadmUpgrade struct { func (k *KubeadmUpgrade) Execute(runtime connector.Runtime) error { host := runtime.RemoteHost() + fmt.Println(k.KubeConf.Cluster.Kubernetes.Version) if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( "timeout -k 600s 600s /usr/local/bin/kubeadm upgrade apply %s -y "+ "--ignore-preflight-errors=all "+ @@ -857,140 +856,128 @@ func SetKubeletTasks(runtime connector.Runtime, kubeAction common.KubeAction) er return nil } -type ReconfigureDNS struct { - common.KubeAction - ModuleName string -} - -func (r *ReconfigureDNS) Execute(runtime connector.Runtime) error { - patchCorednsCmd := `/usr/local/bin/kubectl patch deploy -n kube-system coredns -p \" -spec: - template: - spec: - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile\"` - if _, err := runtime.GetRunner().SudoCmd(patchCorednsCmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "patch the coredns failed") - } - if err := OverrideCoreDNSService(runtime, r.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), "re-config coredns failed") - } - return nil -} - -func OverrideCoreDNSService(runtime connector.Runtime, kubeAction common.KubeAction) error { - host := runtime.RemoteHost() - - generateCoreDNSSvc := &task.RemoteTask{ - Name: "GenerateCoreDNSSvc", - Desc: "generate coredns service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: &action.Template{ - Template: dnsTemplates.CorednsService, - Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.CorednsService.Name()), - Data: util.Data{ - "ClusterIP": kubeAction.KubeConf.Cluster.CorednsClusterIP(), - }, - }, - Parallel: true, - } - - override := &task.RemoteTask{ - Name: "OverrideCoreDNSService", - Desc: "override coredns service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(dns.OverrideCoreDNS), - Parallel: false, - } - - generateNodeLocalDNS := &task.RemoteTask{ - Name: "GenerateNodeLocalDNS", - Desc: "generate nodelocaldns", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - }, - Action: &action.Template{ - Template: dnsTemplates.NodeLocalDNSService, - Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), - Data: util.Data{ - "NodelocaldnsImage": images.GetImage(runtime, kubeAction.KubeConf, "k8s-dns-node-cache").ImageName(), - }, - }, - Parallel: true, - } - - applyNodeLocalDNS := &task.RemoteTask{ - Name: "DeployNodeLocalDNS", - Desc: "deploy nodelocaldns", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - }, - Action: new(dns.DeployNodeLocalDNS), - Parallel: true, - Retry: 5, - } - - generateNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "GenerateNodeLocalDNSConfigMap", - Desc: "generate nodelocaldns configmap", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - new(dns.NodeLocalDNSConfigMapNotExist), - }, - Action: new(dns.GenerateNodeLocalDNSConfigMap), - Parallel: true, - } - - applyNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "ApplyNodeLocalDNSConfigMap", - Desc: "apply nodelocaldns configmap", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - new(dns.NodeLocalDNSConfigMapNotExist), - }, - Action: new(dns.ApplyNodeLocalDNSConfigMap), - Parallel: true, - Retry: 5, - } - - tasks := []task.Interface{ - override, - generateCoreDNSSvc, - override, - generateNodeLocalDNS, - applyNodeLocalDNS, - generateNodeLocalDNSConfigMap, - applyNodeLocalDNSConfigMap, - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} +//type UpgradeDNS struct { +// common.KubeAction +// ModuleName string +//} +// +//func (r *UpgradeDNS) Execute(runtime connector.Runtime) error { +// if err := UpgradeCoredns(runtime, r.KubeAction); err != nil { +// return errors.Wrap(errors.WithStack(err), "re-config coredns failed") +// } +// return nil +//} +// +//func UpgradeCoredns(runtime connector.Runtime, kubeAction common.KubeAction) error { +// host := runtime.RemoteHost() +// +// generateCoreDNSSvc := &task.RemoteTask{ +// Name: "GenerateCoreDNS", +// Desc: "generate coredns manifests", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// }, +// Action: &action.Template{ +// Template: dnsTemplates.Coredns, +// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.Coredns.Name()), +// Data: util.Data{ +// "ClusterIP": kubeAction.KubeConf.Cluster.CorednsClusterIP(), +// "CorednsImage": images.GetImage(runtime, kubeAction.KubeConf, "coredns").ImageName(), +// "DNSEtchHsts": kubeAction.KubeConf.Cluster.DNS.DNSEtcHosts, +// }, +// }, +// Parallel: true, +// } +// +// override := &task.RemoteTask{ +// Name: "UpgradeCoreDNS", +// Desc: "upgrade coredns", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// }, +// Action: new(dns.DeployCoreDNS), +// Parallel: false, +// } +// +// generateNodeLocalDNS := &task.RemoteTask{ +// Name: "GenerateNodeLocalDNS", +// Desc: "generate nodelocaldns", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// new(dns.EnableNodeLocalDNS), +// }, +// Action: &action.Template{ +// Template: dnsTemplates.NodeLocalDNSService, +// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), +// Data: util.Data{ +// "NodelocaldnsImage": images.GetImage(runtime, kubeAction.KubeConf, "k8s-dns-node-cache").ImageName(), +// }, +// }, +// Parallel: true, +// } +// +// applyNodeLocalDNS := &task.RemoteTask{ +// Name: "DeployNodeLocalDNS", +// Desc: "deploy nodelocaldns", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// new(dns.EnableNodeLocalDNS), +// }, +// Action: new(dns.DeployNodeLocalDNS), +// Parallel: true, +// Retry: 5, +// } +// +// generateNodeLocalDNSConfigMap := &task.RemoteTask{ +// Name: "GenerateNodeLocalDNSConfigMap", +// Desc: "generate nodelocaldns configmap", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// new(dns.EnableNodeLocalDNS), +// new(dns.NodeLocalDNSConfigMapNotExist), +// }, +// Action: new(dns.GenerateNodeLocalDNSConfigMap), +// Parallel: true, +// } +// +// applyNodeLocalDNSConfigMap := &task.RemoteTask{ +// Name: "ApplyNodeLocalDNSConfigMap", +// Desc: "apply nodelocaldns configmap", +// Hosts: []connector.Host{host}, +// Prepare: &prepare.PrepareCollection{ +// new(common.OnlyFirstMaster), +// new(dns.EnableNodeLocalDNS), +// new(dns.NodeLocalDNSConfigMapNotExist), +// }, +// Action: new(dns.ApplyNodeLocalDNSConfigMap), +// Parallel: true, +// Retry: 5, +// } +// +// tasks := []task.Interface{ +// override, +// generateCoreDNSSvc, +// override, +// generateNodeLocalDNS, +// applyNodeLocalDNS, +// generateNodeLocalDNSConfigMap, +// applyNodeLocalDNSConfigMap, +// } +// +// for i := range tasks { +// t := tasks[i] +// t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) +// if res := t.Execute(); res.IsFailed() { +// return res.CombineErr() +// } +// } +// return nil +//} type SetCurrentK8sVersion struct { common.KubeAction diff --git a/cmd/kk/pkg/kubernetes/templates/v1beta2/kubeadm_config.go b/cmd/kk/pkg/kubernetes/templates/v1beta2/kubeadm_config.go index e778159e..bef97f24 100644 --- a/cmd/kk/pkg/kubernetes/templates/v1beta2/kubeadm_config.go +++ b/cmd/kk/pkg/kubernetes/templates/v1beta2/kubeadm_config.go @@ -23,7 +23,7 @@ import ( "github.com/lithammer/dedent" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" versionutil "k8s.io/apimachinery/pkg/util/version" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" @@ -33,9 +33,8 @@ import ( ) var ( - funcMap = template.FuncMap{"toYaml": utils.ToYAML, "indent": utils.Indent} // KubeadmConfig defines the template of kubeadm configuration file. - KubeadmConfig = template.Must(template.New("kubeadm-config.yaml").Funcs(funcMap).Parse( + KubeadmConfig = template.Must(template.New("kubeadm-config.yaml").Funcs(utils.FuncMap).Parse( dedent.Dedent(` {{- if .IsInitCluster -}} --- diff --git a/cmd/kk/pkg/kubesphere/tasks.go b/cmd/kk/pkg/kubesphere/tasks.go index a9163b7a..e867fb4e 100644 --- a/cmd/kk/pkg/kubesphere/tasks.go +++ b/cmd/kk/pkg/kubesphere/tasks.go @@ -26,7 +26,7 @@ import ( "time" "github.com/pkg/errors" - yamlV2 "gopkg.in/yaml.v2" + yamlV3 "gopkg.in/yaml.v3" kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" @@ -347,7 +347,7 @@ func (c *ConvertV2ToV3) Execute(runtime connector.Runtime) error { clusterCfgV2 := ksv2.V2{} clusterCfgV3 := ksv3.V3{} - if err := yamlV2.Unmarshal([]byte(configV2Str), &clusterCfgV2); err != nil { + if err := yamlV3.Unmarshal([]byte(configV2Str), &clusterCfgV2); err != nil { return err } @@ -424,7 +424,7 @@ func MigrateConfig2to3(v2 *ksv2.V2, v3 *ksv3.V3) (string, error) { Spec: v3, } - configV3, err := yamlV2.Marshal(clusterConfiguration) + configV3, err := yamlV3.Marshal(clusterConfiguration) if err != nil { return "", err } diff --git a/cmd/kk/pkg/phase/nodes/modules.go b/cmd/kk/pkg/phase/nodes/modules.go index 5b31e927..90532926 100644 --- a/cmd/kk/pkg/phase/nodes/modules.go +++ b/cmd/kk/pkg/phase/nodes/modules.go @@ -64,18 +64,9 @@ func (p *UpgradeNodesModule) Init() { Parallel: false, } - reconfigureDNS := &task.RemoteTask{ - Name: "ReconfigureCoreDNS", - Desc: "Reconfigure CoreDNS", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Action: &kubernetes.ReconfigureDNS{ModuleName: p.Name}, - Parallel: false, - } - p.Tasks = []task.Interface{ upgradeKubeMaster, clusterStatus, upgradeNodes, - reconfigureDNS, } } diff --git a/cmd/kk/pkg/plugins/dns/module.go b/cmd/kk/pkg/plugins/dns/module.go index 09c8cbd1..befb2baa 100644 --- a/cmd/kk/pkg/plugins/dns/module.go +++ b/cmd/kk/pkg/plugins/dns/module.go @@ -36,34 +36,85 @@ func (c *ClusterDNSModule) Init() { c.Name = "ClusterDNSModule" c.Desc = "Deploy cluster dns" - generateCoreDNSSvc := &task.RemoteTask{ - Name: "GenerateCoreDNSSvc", - Desc: "Generate coredns service", + generateCorednsConfigMap := &task.RemoteTask{ + Name: "GenerateCorednsConfigMap", + Desc: "Generate coredns configmap", Hosts: c.Runtime.GetHostsByRole(common.Master), Prepare: &prepare.PrepareCollection{ new(common.OnlyFirstMaster), - &CoreDNSExist{Not: true}, }, Action: &action.Template{ - Template: templates.CorednsService, - Dst: filepath.Join(common.KubeConfigDir, templates.CorednsService.Name()), + Template: templates.CorednsConfigMap, + Dst: filepath.Join(common.KubeConfigDir, templates.CorednsConfigMap.Name()), Data: util.Data{ - "ClusterIP": c.KubeConf.Cluster.CorednsClusterIP(), + "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, + "ExternalZones": c.KubeConf.Cluster.DNS.CoreDNS.ExternalZones, + "AdditionalConfigs": c.KubeConf.Cluster.DNS.CoreDNS.AdditionalConfigs, + "RewriteBlock": c.KubeConf.Cluster.DNS.CoreDNS.RewriteBlock, + "ClusterDomain": c.KubeConf.Cluster.Kubernetes.DNSDomain, + "UpstreamDNSServers": c.KubeConf.Cluster.DNS.CoreDNS.UpstreamDNSServers, }, }, Parallel: true, } - override := &task.RemoteTask{ - Name: "OverrideCoreDNSService", - Desc: "Override coredns service", + applyCorednsConfigMap := &task.RemoteTask{ + Name: "ApplyCorednsConfigMap", + Desc: "Apply coredns configmap", Hosts: c.Runtime.GetHostsByRole(common.Master), Prepare: &prepare.PrepareCollection{ new(common.OnlyFirstMaster), - &CoreDNSExist{Not: true}, }, - Action: new(OverrideCoreDNS), + Action: new(ApplyCorednsConfigMap), Parallel: true, + Retry: 5, + } + + generateCoreDNS := &task.RemoteTask{ + Name: "GenerateCoreDNS", + Desc: "Generate coredns manifests", + Hosts: c.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + }, + Action: new(GenerateCorednsmanifests), + Parallel: true, + } + + deployCoredns := &task.RemoteTask{ + Name: "DeployCoreDNS", + Desc: "Deploy coredns", + Hosts: c.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + }, + Action: new(DeployCoreDNS), + Parallel: true, + } + + generateNodeLocalDNSConfigMap := &task.RemoteTask{ + Name: "GenerateNodeLocalDNSConfigMap", + Desc: "Generate nodelocaldns configmap", + Hosts: c.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + new(EnableNodeLocalDNS), + }, + Action: new(GenerateNodeLocalDNSConfigMap), + Parallel: true, + } + + applyNodeLocalDNSConfigMap := &task.RemoteTask{ + Name: "ApplyNodeLocalDNSConfigMap", + Desc: "Apply nodelocaldns configmap", + Hosts: c.Runtime.GetHostsByRole(common.Master), + Prepare: &prepare.PrepareCollection{ + new(common.OnlyFirstMaster), + new(EnableNodeLocalDNS), + }, + Action: new(ApplyNodeLocalDNSConfigMap), + Parallel: true, + Retry: 5, } generateNodeLocalDNS := &task.RemoteTask{ @@ -79,6 +130,7 @@ func (c *ClusterDNSModule) Init() { Dst: filepath.Join(common.KubeConfigDir, templates.NodeLocalDNSService.Name()), Data: util.Data{ "NodelocaldnsImage": images.GetImage(c.Runtime, c.KubeConf, "k8s-dns-node-cache").ImageName(), + "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, }, }, Parallel: true, @@ -97,39 +149,14 @@ func (c *ClusterDNSModule) Init() { Retry: 5, } - generateNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "GenerateNodeLocalDNSConfigMap", - Desc: "Generate nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - new(NodeLocalDNSConfigMapNotExist), - }, - Action: new(GenerateNodeLocalDNSConfigMap), - Parallel: true, - } - - applyNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "ApplyNodeLocalDNSConfigMap", - Desc: "Apply nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - new(NodeLocalDNSConfigMapNotExist), - }, - Action: new(ApplyNodeLocalDNSConfigMap), - Parallel: true, - Retry: 5, - } - c.Tasks = []task.Interface{ - generateCoreDNSSvc, - override, - generateNodeLocalDNS, - applyNodeLocalDNS, + generateCorednsConfigMap, + applyCorednsConfigMap, + generateCoreDNS, + deployCoredns, generateNodeLocalDNSConfigMap, applyNodeLocalDNSConfigMap, + generateNodeLocalDNS, + applyNodeLocalDNS, } } diff --git a/cmd/kk/pkg/plugins/dns/tasks.go b/cmd/kk/pkg/plugins/dns/tasks.go index 08b815a8..8a336d34 100644 --- a/cmd/kk/pkg/plugins/dns/tasks.go +++ b/cmd/kk/pkg/plugins/dns/tasks.go @@ -17,10 +17,9 @@ package dns import ( - "path/filepath" - "strings" - + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" "github.com/pkg/errors" + "path/filepath" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" @@ -29,19 +28,52 @@ import ( "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" ) -type OverrideCoreDNS struct { +type GenerateCorednsmanifests struct { common.KubeAction } -func (o *OverrideCoreDNS) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl delete -n kube-system svc kube-dns", true); err != nil { - if !strings.Contains(err.Error(), "NotFound") { - return errors.Wrap(errors.WithStack(err), "delete kube-dns failed") - } +func (g *GenerateCorednsmanifests) Execute(runtime connector.Runtime) error { + templateAction := action.Template{ + Template: templates.Coredns, + Dst: filepath.Join(common.KubeConfigDir, templates.Coredns.Name()), + Data: util.Data{ + "ClusterIP": g.KubeConf.Cluster.CorednsClusterIP(), + "CorednsImage": images.GetImage(runtime, g.KubeConf, "coredns").ImageName(), + "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, + }, } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-svc.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "create coredns service failed") + templateAction.Init(nil, nil) + if err := templateAction.Execute(runtime); err != nil { + return err + } + return nil +} + +type DeployCoreDNS struct { + common.KubeAction +} + +func (o *DeployCoreDNS) Execute(runtime connector.Runtime) error { + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl delete svc -n kube-system --field-selector metadata.name=kube-dns", true); err != nil { + return errors.Wrap(errors.WithStack(err), "remove old coredns svc") + } + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns.yaml", true); err != nil { + return errors.Wrap(errors.WithStack(err), "update coredns failed") + } + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl -n kube-system rollout restart deploy coredns", true); err != nil { + return errors.Wrap(errors.WithStack(err), "restart coredns failed") + } + return nil +} + +type ApplyCorednsConfigMap struct { + common.KubeAction +} + +func (o *ApplyCorednsConfigMap) Execute(runtime connector.Runtime) error { + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-configmap.yaml", true); err != nil { + return errors.Wrap(errors.WithStack(err), "create coredns configmap failed") } return nil } @@ -77,6 +109,8 @@ func (g *GenerateNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error Data: util.Data{ "ForwardTarget": clusterIP, "DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain, + "ExternalZones": g.KubeConf.Cluster.DNS.NodeLocalDNS.ExternalZones, + "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, }, } @@ -92,7 +126,7 @@ type ApplyNodeLocalDNSConfigMap struct { } func (a *ApplyNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldnsConfigmap.yaml", true); err != nil { + if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldns-configmap.yaml", true); err != nil { return errors.Wrap(errors.WithStack(err), "apply nodelocaldns configmap failed") } return nil diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns.go b/cmd/kk/pkg/plugins/dns/templates/coredns.go new file mode 100644 index 00000000..4443c153 --- /dev/null +++ b/cmd/kk/pkg/plugins/dns/templates/coredns.go @@ -0,0 +1,211 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package templates + +import ( + "github.com/lithammer/dedent" + "text/template" +) + +var ( + Coredns = template.Must(template.New("coredns.yaml").Parse( + dedent.Dedent(`--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + addonmanager.kubernetes.io/mode: Reconcile + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + createdby: 'kubekey' +spec: + selector: + k8s-app: kube-dns + clusterIP: {{ .ClusterIP }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "coredns" + namespace: kube-system + labels: + k8s-app: "kube-dns" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "coredns" +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 10% + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + createdby: 'kubekey' + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + k8s-app: kube-dns + topologyKey: "kubernetes.io/hostname" + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + containers: + - name: coredns + image: "{{ .CorednsImage }}" + imagePullPolicy: IfNotPresent + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 300Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +{{ if .DNSEtcHosts }} + - key: hosts + path: hosts +{{ end }} + + `))) +) diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go new file mode 100644 index 00000000..fde5f4c3 --- /dev/null +++ b/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go @@ -0,0 +1,97 @@ +/* + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package templates + +import ( + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" + "github.com/lithammer/dedent" + "text/template" +) + +var ( + CorednsConfigMap = template.Must(template.New("coredns-configmap.yaml").Funcs(utils.FuncMap).Parse( + dedent.Dedent(`--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | +{{- if .ExternalZones }} +{{- range .ExternalZones }} +{{ range .Zones }}{{ . | indent 4 }} {{ end }}{ + log + errors +{{- if .Rewrite }} +{{- range .Rewrite }} + rewrite {{ . }} +{{- end }} +{{- end }} + forward .{{ range .Nameservers }} {{ . }}{{ end}} + loadbalance + cache {{ .Cache }} + reload +{{- if $.DNSEtcHosts }} + hosts /etc/coredns/hosts { + fallthrough + } +{{- end }} + } +{{- end }} +{{- end }} + .:53 { +{{- if .AdditionalConfigs }} +{{ .AdditionalConfigs | indent 8 }} +{{- end }} + errors + health { + lameduck 5s + } +{{- if .RewriteBlock }} +{{ .RewriteBlock | indent 8 }} +{{- end }} + ready + kubernetes {{ .ClusterDomain }} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . {{ if .UpstreamDNSServers }}{{ range .UpstreamDNSServers }}{{ . }} {{ end }}{{else}}/etc/resolv.conf{{ end }} { + prefer_udp + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance +{{- if .DNSEtcHosts }} + hosts /etc/coredns/hosts { + fallthrough + } +{{- end }} + } +{{- if .DNSEtcHosts }} + hosts: | +{{ .DNSEtcHosts | indent 4 }} +{{- end }} + + `))) +) diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns_service.go b/cmd/kk/pkg/plugins/dns/templates/coredns_service.go deleted file mode 100644 index b290547a..00000000 --- a/cmd/kk/pkg/plugins/dns/templates/coredns_service.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - CorednsService = template.Must(template.New("coredns-svc.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: Service -metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "coredns" - addonmanager.kubernetes.io/mode: Reconcile - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" -spec: - selector: - k8s-app: kube-dns - clusterIP: {{ .ClusterIP }} - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:coredns -rules: -- apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - watch - - list - - `))) -) diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_service.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go similarity index 95% rename from cmd/kk/pkg/plugins/dns/templates/nodelocaldns_service.go rename to cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go index 330a2bea..6d64ebda 100644 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_service.go +++ b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go @@ -52,6 +52,8 @@ spec: prometheus.io/scrape: 'true' prometheus.io/port: '9253' spec: + nodeSelector: + kubernetes.io/os: linux priorityClassName: system-cluster-critical serviceAccountName: nodelocaldns hostNetwork: true @@ -68,7 +70,7 @@ spec: image: {{ .NodelocaldnsImage }} resources: limits: - memory: 170Mi + memory: 200Mi requests: cpu: 100m memory: 70Mi @@ -115,6 +117,10 @@ spec: items: - key: Corefile path: Corefile +{{- if .DNSEtcHosts }} + - key: hosts + path: hosts +{{ end }} - name: xtables-lock hostPath: path: /run/xtables.lock diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go index 94a055b8..55611500 100644 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go +++ b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go @@ -17,11 +17,12 @@ package templates import ( + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" "github.com/lithammer/dedent" "text/template" ) -var NodeLocalDNSConfigMap = template.Must(template.New("nodelocaldnsConfigmap.yaml").Parse( +var NodeLocalDNSConfigMap = template.Must(template.New("nodelocaldns-configmap.yaml").Funcs(utils.FuncMap).Parse( dedent.Dedent(`--- apiVersion: v1 kind: ConfigMap @@ -33,6 +34,30 @@ metadata: data: Corefile: | +{{- if .ExternalZones }} +{{- range .ExternalZones }} +{{ range .Zones }}{{ . | indent 4 }} {{ end}} { + errors + cache {{ .Cache }} + reload +{{- if .Rewrite }} +{{- range .Rewrite }} + rewrite {{ . }} +{{- end }} +{{- end }} + loop + bind 169.254.25.10 + forward . {{ range .Nameservers }} {{ . }}{{ end }} + prometheus :9253 + log +{{- if $.DNSEtcHosts }} + hosts /etc/coredns/hosts { + fallthrough + } +{{- end }} + } +{{- end }} +{{- end }} {{ .DNSDomain }}:53 { errors cache { @@ -78,6 +103,14 @@ data: bind 169.254.25.10 forward . /etc/resolv.conf prometheus :9253 +{{- if .DNSEtcHosts }} + hosts /etc/coredns/hosts { + fallthrough + } +{{- end }} } - +{{- if .DNSEtcHosts }} + hosts: | +{{ .DNSEtcHosts | indent 4}} +{{- end }} `))) diff --git a/cmd/kk/pkg/utils/util.go b/cmd/kk/pkg/utils/util.go index c03c3c8c..e77b4851 100644 --- a/cmd/kk/pkg/utils/util.go +++ b/cmd/kk/pkg/utils/util.go @@ -20,14 +20,17 @@ import ( "fmt" "regexp" "strings" + "text/template" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" ) +var FuncMap = template.FuncMap{"toYaml": ToYAML, "indent": Indent} + func ResetTmpDir(runtime connector.Runtime) error { _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( "if [ -d %s ]; then rm -rf %s ;fi && mkdir -m 777 -p %s", @@ -50,5 +53,5 @@ func ToYAML(v interface{}) string { func Indent(n int, text string) string { startOfLine := regexp.MustCompile(`(?m)^`) indentation := strings.Repeat(" ", n) - return startOfLine.ReplaceAllLiteralString(text, indentation) + return strings.TrimRight(startOfLine.ReplaceAllLiteralString(text, indentation), " ") } diff --git a/docs/config-example.md b/docs/config-example.md index 61449148..a842760f 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -162,5 +162,71 @@ spec: plainHTTP: false # Allow contacting registries over HTTP. certsPath: "/etc/docker/certs.d/dockerhub.kubekey.local" # Use certificates at path (*.crt, *.cert, *.key) to connect to the registry. addons: [] # You can install cloud-native addons (Chart or YAML) by using this field. + #dns: + # ## Optional hosts file content to coredns use as /etc/hosts file. + # dnsEtcHosts: | + # 192.168.0.100 api.example.com + # 192.168.0.200 ingress.example.com + # coredns: + # ## additionalConfigs adds any extra configuration to coredns + # additionalConfigs: | + # whoami + # log + # ## Array of optional external zones to coredns forward queries to. It's injected into coredns' config file before + # ## default kubernetes zone. Use it as an optimization for well-known zones and/or internal-only domains, i.e. VPN for internal networks (default is unset) + # externalZones: + # - zones: + # - example.com + # - example.io:1053 + # nameservers: + # - 1.1.1.1 + # - 2.2.2.2 + # cache: 5 + # - zones: + # - mycompany.local:4453 + # nameservers: + # - 192.168.0.53 + # cache: 10 + # - zones: + # - mydomain.tld + # nameservers: + # - 10.233.0.3 + # cache: 5 + # rewrite: + # - name substring website.tld website.namespace.svc.cluster.local + # ## Rewrite plugin block to perform internal message rewriting. + # rewriteBlock: | + # rewrite stop { + # name regex (.*)\.my\.domain {1}.svc.cluster.local + # answer name (.*)\.svc\.cluster\.local {1}.my.domain + # } + # ## DNS servers to be added *after* the cluster DNS. These serve as backup + # ## DNS servers in early cluster deployment when no cluster DNS is available yet. + # upstreamDNSServers: + # - 8.8.8.8 + # - 1.2.4.8 + # - 114.114.114.114 + # nodelocaldns: + # ## It's possible to extent the nodelocaldns' configuration by adding an array of external zones. + # externalZones: + # - zones: + # - example.com + # - example.io:1053 + # nameservers: + # - 1.1.1.1 + # - 2.2.2.2 + # cache: 5 + # - zones: + # - mycompany.local:4453 + # nameservers: + # - 192.168.0.53 + # cache: 10 + # - zones: + # - mydomain.tld + # nameservers: + # - 10.233.0.3 + # cache: 5 + # rewrite: + # - name substring website.tld website.namespace.svc.cluster.local ``` diff --git a/go.mod b/go.mod index bececafa..0b0055c8 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 helm.sh/helm/v3 v3.9.4 k8s.io/api v0.25.4 k8s.io/apiextensions-apiserver v0.25.4 @@ -242,7 +242,7 @@ require ( gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect oras.land/oras-go v1.2.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect