feat: auto generate k3s registries.yaml

Signed-off-by: pixiake <guofeng@yunify.com>

pkg/k3s/module.go

@@ -143,6 +143,19 @@ func (i *InitClusterModule) Init() {
 		Parallel: true,
 	}
 
+	k3sRegistryConfig := &task.RemoteTask{
+		Name:  "GenerateK3sRegistryConfig",
+		Desc:  "Generate k3s registry config",
+		Hosts: i.Runtime.GetHostsByRole(common.Master),
+		Prepare: &prepare.PrepareCollection{
+			new(common.OnlyFirstMaster),
+			&ClusterIsExist{Not: true},
+			&UsePrivateRegstry{Not: false},
+		},
+		Action:   new(GenerateK3sRegistryConfig),
+		Parallel: true,
+	}
+
 	enableK3s := &task.RemoteTask{
 		Name:  "EnableK3sService",
 		Desc:  "Enable k3s service",
@@ -198,6 +211,7 @@ func (i *InitClusterModule) Init() {
 	i.Tasks = []task.Interface{
 		k3sService,
 		k3sEnv,
+		k3sRegistryConfig,
 		enableK3s,
 		copyKubeConfig,
 		addMasterTaint,
@@ -235,6 +249,18 @@ func (j *JoinNodesModule) Init() {
 		Parallel: true,
 	}
 
+	k3sRegistryConfig := &task.RemoteTask{
+		Name:  "GenerateK3sRegistryConfig",
+		Desc:  "Generate k3s registry config",
+		Hosts: j.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&NodeInCluster{Not: true},
+			&UsePrivateRegstry{Not: false},
+		},
+		Action:   new(GenerateK3sRegistryConfig),
+		Parallel: true,
+	}
+
 	enableK3s := &task.RemoteTask{
 		Name:  "EnableK3sService",
 		Desc:  "Enable k3s service",
@@ -298,6 +324,7 @@ func (j *JoinNodesModule) Init() {
 	j.Tasks = []task.Interface{
 		k3sService,
 		k3sEnv,
+		k3sRegistryConfig,
 		enableK3s,
 		copyKubeConfigForMaster,
 		syncKubeConfigToWorker,

pkg/k3s/prepares.go

@@ -59,3 +59,12 @@ func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) {
 		return false, errors.New("get k3s cluster status by pipeline cache failed")
 	}
 }
+
+type UsePrivateRegstry struct {
+	common.KubePrepare
+	Not bool
+}
+
+func (c *UsePrivateRegstry) PreCheck(_ connector.Runtime) (bool, error) {
+	return c.KubeConf.Cluster.Registry.PrivateRegistry != "", nil
+}

pkg/k3s/tasks.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"github.com/kubesphere/kubekey/pkg/registry"
 	"path/filepath"
 	"strings"
 
 	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2"
+	kubekeyregistry "github.com/kubesphere/kubekey/pkg/bootstrap/registry"
 	"github.com/kubesphere/kubekey/pkg/common"
 	"github.com/kubesphere/kubekey/pkg/core/action"
 	"github.com/kubesphere/kubekey/pkg/core/connector"
@@ -508,3 +510,68 @@ func (s *SaveKubeConfig) Execute(_ connector.Runtime) error {
 	}
 	return nil
 }
+
+type GenerateK3sRegistryConfig struct {
+	common.KubeAction
+}
+
+func (g *GenerateK3sRegistryConfig) Execute(runtime connector.Runtime) error {
+	endpointPrefix := "https://"
+	dockerioMirror := registry.Mirror{}
+	registryConfigs := map[string]registry.RegistryConfig{}
+
+	auths := registry.DockerRegistryAuthEntries(g.KubeConf.Cluster.Registry.Auths)
+	for k, v := range auths {
+		if k == g.KubeConf.Cluster.Registry.PrivateRegistry && v.PlainHTTP {
+			endpointPrefix = "http://"
+		}
+	}
+
+	dockerioMirror.Endpoints = []string{fmt.Sprintf("%s%s", endpointPrefix, g.KubeConf.Cluster.Registry.PrivateRegistry)}
+
+	if g.KubeConf.Cluster.Registry.NamespaceOverride != "" {
+		dockerioMirror.Rewrites = map[string]string{
+			"^rancher/(.*)": fmt.Sprintf("%s/$1", g.KubeConf.Cluster.Registry.NamespaceOverride),
+		}
+	}
+
+	for k, v := range auths {
+		registryConfigs[k] = registry.RegistryConfig{
+			Auth: &registry.AuthConfig{
+				Username: v.Username,
+				Password: v.Password,
+			},
+			TLS: &registry.TLSConfig{
+				CAFile:             v.CAFile,
+				CertFile:           v.CertFile,
+				KeyFile:            v.KeyFile,
+				InsecureSkipVerify: v.SkipTLSVerify,
+			},
+		}
+	}
+
+	_, ok := registryConfigs[kubekeyregistry.RegistryCertificateBaseName]
+
+	if !ok && g.KubeConf.Cluster.Registry.PrivateRegistry == kubekeyregistry.RegistryCertificateBaseName {
+		registryConfigs[g.KubeConf.Cluster.Registry.PrivateRegistry] = registry.RegistryConfig{TLS: &registry.TLSConfig{InsecureSkipVerify: true}}
+	}
+
+	k3sRegistries := registry.Registry{
+		Mirrors: map[string]registry.Mirror{"docker.io": dockerioMirror},
+		Configs: registryConfigs,
+	}
+
+	templateAction := action.Template{
+		Template: templates.K3sRegistryConfigTempl,
+		Dst:      filepath.Join("/etc/rancher/k3s", templates.K3sRegistryConfigTempl.Name()),
+		Data: util.Data{
+			"Registries": k3sRegistries,
+		},
+	}
+
+	templateAction.Init(nil, nil)
+	if err := templateAction.Execute(runtime); err != nil {
+		return err
+	}
+	return nil
+}

pkg/k3s/templates/k3sRegistryConfig.go

@@ -0,0 +1,30 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package templates
+
+import (
+	"github.com/kubesphere/kubekey/pkg/utils"
+	"github.com/lithammer/dedent"
+	"text/template"
+)
+
+var (
+	funcMap = template.FuncMap{"toYaml": utils.ToYAML, "indent": utils.Indent}
+	// k3sRegistryConfigTempl defines the template of k3s' registry.
+	K3sRegistryConfigTempl = template.Must(template.New("registries.yaml").Funcs(funcMap).Parse(
+		dedent.Dedent(`{{ toYaml .Registries }}`)))
+)

pkg/kubernetes/templates/v1beta2/kubeadm_config.go

@@ -18,7 +18,7 @@ package v1beta2
 
 import (
 	"fmt"
-	"regexp"
+	"github.com/kubesphere/kubekey/pkg/utils"
 	"strings"
 	"text/template"
 
@@ -34,7 +34,7 @@ import (
 )
 
 var (
-	funcMap = template.FuncMap{"toYaml": toYAML, "indent": Indent}
+	funcMap = template.FuncMap{"toYaml": utils.ToYAML, "indent": utils.Indent}
 	// KubeadmConfig defines the template of kubeadm configuration file.
 	KubeadmConfig = template.Must(template.New("kubeadm-config.yaml").Funcs(funcMap).Parse(
 		dedent.Dedent(`
@@ -363,18 +363,3 @@ func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{}
 
 	return kubeProxyConfiguration
 }
-
-func toYAML(v interface{}) string {
-	data, err := yaml.Marshal(v)
-	if err != nil {
-		// Swallow errors inside of a template.
-		return ""
-	}
-	return strings.TrimSuffix(string(data), "\n")
-}
-
-func Indent(n int, text string) string {
-	startOfLine := regexp.MustCompile(`(?m)^`)
-	indentation := strings.Repeat(" ", n)
-	return startOfLine.ReplaceAllLiteralString(text, indentation)
-}

pkg/registry/k3s_registries.go

@@ -0,0 +1,60 @@
+package registry
+
+// Mirror contains the config related to the registry mirror
+type Mirror struct {
+	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
+	// one by one until a working one is found. The endpoint must be a valid url
+	// with host specified.
+	// The scheme, host and path from the endpoint URL will be used.
+	Endpoints []string `toml:"endpoint" yaml:"endpoint"`
+
+	// Rewrites are repository rewrite rules for a namespace. When fetching image resources
+	// from an endpoint and a key matches the repository via regular expression matching
+	// it will be replaced with the corresponding value from the map in the resource request.
+	Rewrites map[string]string `toml:"rewrite" yaml:"rewrite"`
+}
+
+// AuthConfig contains the config related to authentication to a specific registry
+type AuthConfig struct {
+	// Username is the username to login the registry.
+	Username string `toml:"username" yaml:"username"`
+	// Password is the password to login the registry.
+	Password string `toml:"password" yaml:"password"`
+	// Auth is a base64 encoded string from the concatenation of the username,
+	// a colon, and the password.
+	Auth string `toml:"auth" yaml:"auth"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `toml:"identitytoken" yaml:"identity_token"`
+}
+
+// TLSConfig contains the CA/Cert/Key used for a registry
+type TLSConfig struct {
+	CAFile             string `toml:"ca_file" yaml:"ca_file"`
+	CertFile           string `toml:"cert_file" yaml:"cert_file"`
+	KeyFile            string `toml:"key_file" yaml:"key_file"`
+	InsecureSkipVerify bool   `toml:"insecure_skip_verify" yaml:"insecure_skip_verify"`
+}
+
+// Registry is registry settings including mirrors, TLS, and credentials
+type Registry struct {
+	// Mirrors are namespace to mirror mapping for all namespaces.
+	Mirrors map[string]Mirror `toml:"mirrors" yaml:"mirrors"`
+	// Configs are configs for each registry.
+	// The key is the FDQN or IP of the registry.
+	Configs map[string]RegistryConfig `toml:"configs" yaml:"configs"`
+
+	// Auths are registry endpoint to auth config mapping. The registry endpoint must
+	// be a valid url with host specified.
+	// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
+	Auths map[string]AuthConfig `toml:"auths" yaml:"auths"`
+}
+
+// RegistryConfig contains configuration used to communicate with the registry.
+type RegistryConfig struct {
+	// Auth contains information to authenticate to the registry.
+	Auth *AuthConfig `toml:"auth" yaml:"auth"`
+	// TLS is a pair of CA/Cert/Key which then are used when creating the transport
+	// that communicates with the registry.
+	TLS *TLSConfig `toml:"tls" yaml:"tls"`
+}

pkg/utils/util.go

@@ -21,6 +21,9 @@ import (
 	"github.com/kubesphere/kubekey/pkg/common"
 	"github.com/kubesphere/kubekey/pkg/core/connector"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	"regexp"
+	"strings"
 )
 
 func ResetTmpDir(runtime connector.Runtime) error {
@@ -32,3 +35,18 @@ func ResetTmpDir(runtime connector.Runtime) error {
 	}
 	return nil
 }
+
+func ToYAML(v interface{}) string {
+	data, err := yaml.Marshal(v)
+	if err != nil {
+		// Swallow errors inside of a template.
+		return ""
+	}
+	return strings.TrimSuffix(string(data), "\n")
+}
+
+func Indent(n int, text string) string {
+	startOfLine := regexp.MustCompile(`(?m)^`)
+	indentation := strings.Repeat(" ", n)
+	return startOfLine.ReplaceAllLiteralString(text, indentation)
+}