15308555518/ kubekey
1
0
Fork 0
mirror of https://github.com/kubesphere/kubekey.git synced 2025-12-26 01:22:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'kubesphere:master' into master

This commit is contained in:
Tan Guofu 2022-01-08 14:28:27 +08:00 committed by GitHub
parent 92695eea77 cbdef5a7a4
commit a8571fd584
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1455 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
apis/kubekey/v1alpha2/cluster_types.go
View File

@ -148,28 +148,31 @@ type HostCfg struct {
PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"`
Arch string `yaml:"arch,omitempty" json:"arch,omitempty"`
Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
ID string `yaml:"id,omitempty" json:"id,omitempty"`
Index int `json:"-"`
IsEtcd bool `json:"-"`
IsMaster bool `json:"-"`
IsWorker bool `json:"-"`
Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
ID string `yaml:"id,omitempty" json:"id,omitempty"`
Index int `json:"-"`
IsEtcd bool `json:"-"`
IsMaster bool `json:"-"`
IsWorker bool `json:"-"`
IsRegistry bool `json:"-"`
}
// RoleGroups defines the grouping of role for hosts (etcd / master / worker).
// RoleGroups defines the grouping of role for hosts (etcd / master / worker / registry).
type RoleGroups struct {
Etcd []string `yaml:"etcd" json:"etcd,omitempty"`
Master []string `yaml:"master" json:"master,omitempty"`
Worker []string `yaml:"worker" json:"worker,omitempty"`
Etcd []string `yaml:"etcd" json:"etcd,omitempty"`
Master []string `yaml:"master" json:"master,omitempty"`
Worker []string `yaml:"worker" json:"worker,omitempty"`
Registry []string `yaml:"registry" json:"registry,omitempty"`
}
// HostGroups defines the grouping of hosts for cluster (all / etcd / master / worker / k8s).
type HostGroups struct {
All []HostCfg
Etcd []HostCfg
Master []HostCfg
Worker []HostCfg
K8s []HostCfg
All []HostCfg
Etcd []HostCfg
Master []HostCfg
Worker []HostCfg
K8s []HostCfg
Registry []HostCfg
}
// ControlPlaneEndpoint defines the control plane endpoint information for cluster.
@ -188,6 +191,7 @@ type System struct {
// RegistryConfig defines the configuration information of the image's repository.
type RegistryConfig struct {
Type string `yaml:"type" json:"type,omitempty"`
RegistryMirrors []string `yaml:"registryMirrors" json:"registryMirrors,omitempty"`
InsecureRegistries []string `yaml:"insecureRegistries" json:"insecureRegistries,omitempty"`
PrivateRegistry string `yaml:"privateRegistry" json:"privateRegistry,omitempty"`
@ -249,7 +253,7 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
hostList[host.Name] = host.Name
}
etcdGroup, masterGroup, workerGroup, err := cfg.ParseRolesList(hostList)
etcdGroup, masterGroup, workerGroup, registryGroup, err := cfg.ParseRolesList(hostList)
if err != nil {
return nil, err
}
@ -282,6 +286,14 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
}
}
if len(registryGroup) > 0 {
for _, hostName := range registryGroup {
if hostName != "" && host.Name == hostName {
host.IsRegistry = true
}
}
}
if host.IsEtcd {
clusterHostsGroups.Etcd = append(clusterHostsGroups.Etcd, host)
}
@ -294,6 +306,10 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
if host.IsMaster || host.IsWorker {
clusterHostsGroups.K8s = append(clusterHostsGroups.K8s, host)
}
if host.IsRegistry {
clusterHostsGroups.Registry = append(clusterHostsGroups.Registry, host)
}
clusterHostsGroups.All = append(clusterHostsGroups.All, host)
}
@ -304,6 +320,9 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
if len(etcdGroup) == 0 {
logger.Log.Fatal(errors.New("The number of etcd cannot be 0"))
}
if len(registryGroup) > 1 {
logger.Log.Fatal(errors.New("The number of registry node cannot be greater than 1."))
}
if len(masterGroup) != len(clusterHostsGroups.Master) {
return nil, errors.New("Incorrect nodeName under roleGroups/master in the configuration file")
@ -314,6 +333,9 @@ func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
if len(workerGroup) != len(clusterHostsGroups.Worker) {
return nil, errors.New("Incorrect nodeName under roleGroups/work in the configuration file")
}
if len(registryGroup) != len(clusterHostsGroups.Registry) {
return nil, errors.New("Incorrect nodeName under roleGroups/registry in the configuration file")
}
return &clusterHostsGroups, nil
}
@ -338,10 +360,11 @@ func (cfg *ClusterSpec) ClusterDNS() string {
}
// ParseRolesList is used to parse the host grouping list.
func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []string, []string, error) {
func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []string, []string, []string, error) {
etcdGroupList := make([]string, 0)
masterGroupList := make([]string, 0)
workerGroupList := make([]string, 0)
registryGroupList := make([]string, 0)
for _, host := range cfg.RoleGroups.Etcd {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
@ -375,7 +398,19 @@ func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []
workerGroupList = append(workerGroupList, host)
}
}
return etcdGroupList, masterGroupList, workerGroupList, nil
for _, host := range cfg.RoleGroups.Registry {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
registryGroupList = append(registryGroupList, getHostsRange(host, hostList, "registry")...)
} else {
if err := hostVerify(hostList, host, "registry"); err != nil {
logger.Log.Fatal(err)
}
registryGroupList = append(registryGroupList, host)
}
}
return etcdGroupList, masterGroupList, workerGroupList, registryGroupList, nil
}
func getHostsRange(rangeStr string, hostList map[string]string, group string) []string {

97
apis/kubekey/v1alpha2/default.go
View File

@ -24,53 +24,56 @@ import (
)
const (
DefaultPreDir = "kubekey"
DefaultTmpDir = "/tmp/kubekey"
DefaultSSHPort = 22
DefaultLBPort = 6443
DefaultLBDomain = "lb.kubesphere.local"
DefaultNetworkPlugin = "calico"
DefaultPodsCIDR = "10.233.64.0/18"
DefaultServiceCIDR = "10.233.0.0/18"
DefaultKubeImageNamespace = "kubesphere"
DefaultClusterName = "cluster.local"
DefaultArch = "amd64"
DefaultEtcdVersion = "v3.4.13"
DefaultEtcdPort = "2379"
DefaultDockerVersion = "20.10.8"
DefaultCrictlVersion = "v1.22.0"
DefaultKubeVersion = "v1.21.5"
DefaultCalicoVersion = "v3.20.0"
DefaultFlannelVersion = "v0.12.0"
DefaultCniVersion = "v0.9.1"
DefaultCiliumVersion = "v1.8.3"
DefaultKubeovnVersion = "v1.5.0"
DefalutMultusVersion = "v3.8"
DefaultHelmVersion = "v3.6.3"
DefaultMaxPods = 110
DefaultNodeCidrMaskSize = 24
DefaultIPIPMode = "Always"
DefaultVXLANMode = "Never"
DefaultVethMTU = 1440
DefaultBackendMode = "vxlan"
DefaultProxyMode = "ipvs"
DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock"
DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock"
DefaultIsulaEndpoint = "unix:///var/run/isulad.sock"
Etcd = "etcd"
Master = "master"
Worker = "worker"
K8s = "k8s"
DefaultEtcdBackupDir = "/var/backups/kube_etcd"
DefaultEtcdBackupPeriod = 30
DefaultKeepBackNumber = 5
DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts"
DefaultJoinCIDR = "100.64.0.0/16"
DefaultNetworkType = "geneve"
DefaultVlanID = "100"
DefaultOvnLabel = "node-role.kubernetes.io/master"
DefaultDPDKVersion = "19.11"
DefaultDNSAddress = "114.114.114.114"
DefaultPreDir = "kubekey"
DefaultTmpDir = "/tmp/kubekey"
DefaultSSHPort = 22
DefaultLBPort = 6443
DefaultLBDomain = "lb.kubesphere.local"
DefaultNetworkPlugin = "calico"
DefaultPodsCIDR = "10.233.64.0/18"
DefaultServiceCIDR = "10.233.0.0/18"
DefaultKubeImageNamespace = "kubesphere"
DefaultClusterName = "cluster.local"
DefaultArch = "amd64"
DefaultEtcdVersion = "v3.4.13"
DefaultEtcdPort = "2379"
DefaultDockerVersion = "20.10.8"
DefaultCrictlVersion = "v1.22.0"
DefaultKubeVersion = "v1.21.5"
DefaultCalicoVersion = "v3.20.0"
DefaultFlannelVersion = "v0.12.0"
DefaultCniVersion = "v0.9.1"
DefaultCiliumVersion = "v1.8.3"
DefaultKubeovnVersion = "v1.5.0"
DefalutMultusVersion = "v3.8"
DefaultHelmVersion = "v3.6.3"
DefaultDockerComposeVersion = "v2.2.2"
DefaultRegistryVersion = "2"
DefaultHarborVersion = "v2.4.1"
DefaultMaxPods = 110
DefaultNodeCidrMaskSize = 24
DefaultIPIPMode = "Always"
DefaultVXLANMode = "Never"
DefaultVethMTU = 1440
DefaultBackendMode = "vxlan"
DefaultProxyMode = "ipvs"
DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock"
DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock"
DefaultIsulaEndpoint = "unix:///var/run/isulad.sock"
Etcd = "etcd"
Master = "master"
Worker = "worker"
K8s = "k8s"
DefaultEtcdBackupDir = "/var/backups/kube_etcd"
DefaultEtcdBackupPeriod = 30
DefaultKeepBackNumber = 5
DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts"
DefaultJoinCIDR = "100.64.0.0/16"
DefaultNetworkType = "geneve"
DefaultVlanID = "100"
DefaultOvnLabel = "node-role.kubernetes.io/master"
DefaultDPDKVersion = "19.11"
DefaultDNSAddress = "114.114.114.114"
Docker = "docker"
Conatinerd = "containerd"

12
apis/kubekey/v1alpha2/zz_generated.deepcopy.go
View File

@ -480,6 +480,13 @@ func (in *HostGroups) DeepCopyInto(out *HostGroups) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Registry != nil {
in, out := &in.Registry, &out.Registry
*out = make([]HostCfg, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostGroups.
@ -890,6 +897,11 @@ func (in *RoleGroups) DeepCopyInto(out *RoleGroups) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Registry != nil {
in, out := &in.Registry, &out.Registry
*out = make([]string, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleGroups.

1
cmd/ctl/init/init.go
View File

@ -40,5 +40,6 @@ func NewCmdInit() *cobra.Command {
}
o.CommonOptions.AddCommonFlag(cmd)
cmd.AddCommand(NewCmdInitOs())
cmd.AddCommand(NewCmdInitRegistry())
return cmd
}

67
cmd/ctl/init/init_registry.go Normal file
View File

@ -0,0 +1,67 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package init
import (
"github.com/kubesphere/kubekey/cmd/ctl/options"
"github.com/kubesphere/kubekey/cmd/ctl/util"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/pipelines"
"github.com/spf13/cobra"
)
type InitRegistryOptions struct {
CommonOptions *options.CommonOptions
ClusterCfgFile string
DownloadCmd string
}
func NewInitRegistryOptions() *InitRegistryOptions {
return &InitRegistryOptions{
CommonOptions: options.NewCommonOptions(),
}
}
// NewCmdInitRegistry creates a new init os command
func NewCmdInitRegistry() *cobra.Command {
o := NewInitRegistryOptions()
cmd := &cobra.Command{
Use: "registry",
Short: "Init a local image registry",
Run: func(cmd *cobra.Command, args []string) {
util.CheckErr(o.Run())
},
}
o.CommonOptions.AddCommonFlag(cmd)
o.AddFlags(cmd)
return cmd
}
func (o *InitRegistryOptions) Run() error {
arg := common.Argument{
FilePath: o.ClusterCfgFile,
Debug: o.CommonOptions.Verbose,
}
return pipelines.InitRegistry(arg, o.DownloadCmd)
}
func (o *InitRegistryOptions) AddFlags(cmd *cobra.Command) {
cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file")
cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
`The user defined command to download the necessary files. The first param '%s' is output path, the second param '%s', is the URL`)
}

8
config/crd/bases/kubekey.kubesphere.io_clusters.yaml
View File

@ -608,10 +608,12 @@ spec:
items:
type: string
type: array
type:
type: string
type: object
roleGroups:
description: RoleGroups defines the grouping of role for hosts (etcd
/ master / worker).
/ master / worker / registry).
properties:
etcd:
items:
@ -621,6 +623,10 @@ spec:
items:
type: string
type: array
registry:
items:
type: string
type: array
worker:
items:
type: string

2
go.mod
View File

@ -3,7 +3,7 @@ module github.com/kubesphere/kubekey
go 1.17
require (
github.com/containerd/containerd v1.5.8
github.com/containerd/containerd v1.5.9
github.com/deckarep/golang-set v1.7.1
github.com/dominodatalab/os-release v0.0.0-20190522011736-bcdb4a3e3c2f
github.com/go-logr/logr v1.2.2

4
go.sum
View File

@ -232,8 +232,8 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT
github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
github.com/containerd/containerd v1.5.9 h1:rs6Xg1gtIxaeyG+Smsb/0xaSDu1VgFhOCKBXxMxbsF4=
github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=

22
pkg/artifact/module.go
View File

@ -120,13 +120,29 @@ func (u *UnArchiveModule) Init() {
u.Name = "UnArchiveArtifactModule"
u.Desc = "UnArchive the KubeKey artifact"
md5Check := &task.LocalTask{
Name: "CheckArtifactMd5",
Desc: "Check the KubeKey artifact md5 value",
Action: new(Md5Check),
}
unArchive := &task.LocalTask{
Name: "UnArchiveArtifact",
Desc: "UnArchive the KubeKey artifact",
Action: new(UnArchive),
Name: "UnArchiveArtifact",
Desc: "UnArchive the KubeKey artifact",
Prepare: &Md5AreEqual{Not: true},
Action: new(UnArchive),
}
createMd5File := &task.LocalTask{
Name: "CreateArtifactMd5File",
Desc: "Create the KubeKey artifact Md5 file",
Prepare: &Md5AreEqual{Not: true},
Action: new(CreateMd5File),
}
u.Tasks = []task.Interface{
md5Check,
unArchive,
createMd5File,
}
}

18
pkg/artifact/prepares.go
View File

@ -17,6 +17,7 @@
package artifact
import (
"fmt"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/connector"
)
@ -33,3 +34,20 @@ func (e *EnableDownload) PreCheck(_ connector.Runtime) (bool, error) {
}
return false, nil
}
type Md5AreEqual struct {
common.KubePrepare
Not bool
}
func (m *Md5AreEqual) PreCheck(_ connector.Runtime) (bool, error) {
equal, ok := m.ModuleCache.GetMustBool("md5AreEqual")
if !ok {
return false, fmt.Errorf("get md5 equal value from module cache failed")
}
if equal {
return !m.Not, nil
}
return m.Not, nil
}

45
pkg/artifact/tasks.go
View File

@ -32,6 +32,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@ -274,3 +275,47 @@ func (u *UnArchive) Execute(runtime connector.Runtime) error {
}
return nil
}
type Md5Check struct {
common.KubeAction
}
func (m *Md5Check) Execute(runtime connector.Runtime) error {
m.ModuleCache.Set("md5AreEqual", false)
// check if there is a md5.sum file. This file's content contains the last artifact md5 value.
oldFile := filepath.Join(runtime.GetWorkDir(), "md5.sum")
if exist := coreutil.IsExist(oldFile); !exist {
return nil
}
oldMd5, err := ioutil.ReadFile(oldFile)
if err != nil {
return errors.Wrapf(errors.WithStack(err), "read old md5 file %s failed", oldFile)
}
newMd5 := coreutil.LocalMd5Sum(m.KubeConf.Arg.Artifact)
if string(oldMd5) == newMd5 {
m.ModuleCache.Set("md5AreEqual", true)
}
return nil
}
type CreateMd5File struct {
common.KubeAction
}
func (c *CreateMd5File) Execute(runtime connector.Runtime) error {
oldFile := filepath.Join(runtime.GetWorkDir(), "md5.sum")
newMd5 := coreutil.LocalMd5Sum(c.KubeConf.Arg.Artifact)
f, err := os.Create(oldFile)
if err != nil {
return errors.Wrapf(errors.WithStack(err), "create md5 fild %s failed", oldFile)
}
if _, err := io.Copy(f, strings.NewReader(newMd5)); err != nil {
return errors.Wrapf(errors.WithStack(err), "write md5 value to file %s failed", oldFile)
}
return nil
}

4
pkg/binaries/k3s.go
View File

@ -70,7 +70,7 @@ func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str
binaries := []files.KubeBinary{k3s, helm, kubecni, etcd}
binariesMap := make(map[string]files.KubeBinary)
for _, binary := range binaries {
logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name)
logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name)
binariesMap[binary.Name] = binary
if util.IsExist(binary.Path) {
@ -122,7 +122,7 @@ func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str
}
}
pipelineCache.Set(common.KubeBinaries, binariesMap)
pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap)
return nil
}

11
pkg/binaries/kubernetes.go
View File

@ -88,7 +88,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str
binaries := []files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, docker, crictl, etcd}
binariesMap := make(map[string]files.KubeBinary)
for _, binary := range binaries {
logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name)
logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name)
binariesMap[binary.Name] = binary
if util.IsExist(binary.Path) {
@ -151,7 +151,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, filepath, version, arch str
}
}
pipelineCache.Set(common.KubeBinaries, binariesMap)
pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap)
return nil
}
@ -185,7 +185,7 @@ func sha256sum(path string) (string, error) {
return fmt.Sprintf("%x", sha256.Sum256(data)), nil
}
func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch string, pipelineCache *cache.Cache) error {
func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error {
kkzone := os.Getenv("KKZONE")
m := manifest.Spec
@ -220,11 +220,9 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path,
binaries = append(binaries, crictl)
}
binariesMap := make(map[string]files.KubeBinary)
for _, binary := range binaries {
logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name)
logger.Log.Messagef(common.LocalHost, "downloading %s %s ...", arch, binary.Name)
binariesMap[binary.Name] = binary
if util.IsExist(binary.Path) {
// download it again if it's incorrect
if err := SHA256Check(binary); err != nil {
@ -274,6 +272,5 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path,
}
}
pipelineCache.Set(common.KubeBinaries, binariesMap)
return nil
}

25
pkg/binaries/module.go
View File

@ -18,7 +18,9 @@ package binaries
import (
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/logger"
"github.com/kubesphere/kubekey/pkg/core/task"
"github.com/pkg/errors"
)
type NodeBinariesModule struct {
@ -77,3 +79,26 @@ func (a *ArtifactBinariesModule) Init() {
download,
}
}
type RegistryPackageModule struct {
common.KubeModule
}
func (n *RegistryPackageModule) Init() {
n.Name = "RegistryPackageModule"
n.Desc = "Download registry package"
if len(n.Runtime.GetHostsByRole(common.Registry)) == 0 {
logger.Log.Fatal(errors.New("[registry] node not found in the roleGroups of the configuration file"))
}
download := &task.LocalTask{
Name: "DownloadRegistryPackage",
Desc: "Download registry package",
Action: new(RegistryPackageDownload),
}
n.Tasks = []task.Interface{
download,
}
}

131
pkg/binaries/registry.go Normal file
View File

@ -0,0 +1,131 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package binaries
import (
"fmt"
kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/cache"
"github.com/kubesphere/kubekey/pkg/core/logger"
"github.com/kubesphere/kubekey/pkg/core/util"
"github.com/kubesphere/kubekey/pkg/files"
"github.com/pkg/errors"
"io"
"os"
"os/exec"
)
// RegistryPackageDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them.
func RegistryPackageDownloadHTTP(kubeConf *common.KubeConf, filepath, arch string, pipelineCache *cache.Cache) error {
kkzone := os.Getenv("KKZONE")
binaries := []files.KubeBinary{}
switch kubeConf.Cluster.Registry.Type {
case common.Harbor:
harbor := files.KubeBinary{Name: "harbor", Arch: arch, Version: kubekeyapiv1alpha2.DefaultHarborVersion}
harbor.Path = fmt.Sprintf("%s/harbor-offline-installer-%s.tgz", filepath, kubekeyapiv1alpha2.DefaultHarborVersion)
// TODO: Harbor only supports amd64, so there is no need to consider other architectures at present.
docker := files.KubeBinary{Name: "docker", Arch: arch, Version: kubekeyapiv1alpha2.DefaultDockerVersion}
docker.Path = fmt.Sprintf("%s/docker-%s.tgz", filepath, kubekeyapiv1alpha2.DefaultDockerVersion)
compose := files.KubeBinary{Name: "compose", Arch: arch, Version: kubekeyapiv1alpha2.DefaultDockerComposeVersion}
compose.Path = fmt.Sprintf("%s/docker-compose-linux-x86_64", filepath)
if kkzone == "cn" {
harbor.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", harbor.Version, harbor.Version)
docker.Url = fmt.Sprintf("https://mirrors.aliyun.com/docker-ce/linux/static/stable/x86_64/docker-%s.tgz", docker.Version)
compose.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/%s/docker-compose-linux-x86_64", compose.Version)
} else {
harbor.Url = fmt.Sprintf("https://github.com/goharbor/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", harbor.Version, harbor.Version)
docker.Url = fmt.Sprintf("https://download.docker.com/linux/static/stable/x86_64/docker-%s.tgz", docker.Version)
compose.Url = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/docker-compose-linux-x86_64", compose.Version)
}
harbor.GetCmd = kubeConf.Arg.DownloadCommand(harbor.Path, harbor.Url)
docker.GetCmd = kubeConf.Arg.DownloadCommand(docker.Path, docker.Url)
compose.GetCmd = kubeConf.Arg.DownloadCommand(compose.Path, compose.Url)
binaries = []files.KubeBinary{harbor, docker, compose}
default:
registry := files.KubeBinary{Name: "registry", Arch: arch, Version: kubekeyapiv1alpha2.DefaultRegistryVersion}
registry.Path = fmt.Sprintf("%s/registry-%s-linux-%s.tar.gz", filepath, kubekeyapiv1alpha2.DefaultRegistryVersion, arch)
if kkzone == "cn" {
registry.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/registry/%s/registry-%s-linux-%s.tar.gz", kubekeyapiv1alpha2.DefaultRegistryVersion, kubekeyapiv1alpha2.DefaultRegistryVersion, registry.Arch)
} else {
registry.Url = fmt.Sprintf("https://github.com/kubesphere/kubekey/releases/download/v2.0.0-alpha.1/registry-%s-linux-%s.tar.gz", kubekeyapiv1alpha2.DefaultRegistryVersion, registry.Arch)
}
registry.GetCmd = kubeConf.Arg.DownloadCommand(registry.Path, registry.Url)
binaries = []files.KubeBinary{registry}
}
binariesMap := make(map[string]files.KubeBinary)
for _, binary := range binaries {
logger.Log.Messagef(common.LocalHost, "downloading %s ...", binary.Name)
binariesMap[binary.Name] = binary
if util.IsExist(binary.Path) {
// download it again if it's incorrect
if err := SHA256Check(binary); err != nil {
_ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path)).Run()
} else {
continue
}
}
for i := 5; i > 0; i-- {
cmd := exec.Command("/bin/sh", "-c", binary.GetCmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err)
}
cmd.Stderr = cmd.Stdout
if err = cmd.Start(); err != nil {
return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err)
}
for {
tmp := make([]byte, 1024)
_, err := stdout.Read(tmp)
fmt.Print(string(tmp)) // Get the output from the pipeline in real time and print it to the terminal
if errors.Is(err, io.EOF) {
break
} else if err != nil {
logger.Log.Errorln(err)
break
}
}
if err = cmd.Wait(); err != nil {
if kkzone != "cn" {
logger.Log.Warningln("Having a problem with accessing https://storage.googleapis.com? You can try again after setting environment 'export KKZONE=cn'")
}
return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.Name, binary.GetCmd, err)
}
if err := SHA256Check(binary); err != nil {
if i == 1 {
return err
}
_ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path)).Run()
continue
}
break
}
}
pipelineCache.Set(common.KubeBinaries, binariesMap)
return nil
}

20
pkg/binaries/tasks.go
View File

@ -129,9 +129,27 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error {
return errors.Wrap(err, "Failed to create download target dir")
}
if err := KubernetesArtifactBinariesDownload(a.Manifest, binariesDir, arch, a.PipelineCache); err != nil {
if err := KubernetesArtifactBinariesDownload(a.Manifest, binariesDir, arch); err != nil {
return err
}
}
return nil
}
type RegistryPackageDownload struct {
common.KubeAction
}
func (k *RegistryPackageDownload) Execute(runtime connector.Runtime) error {
arch := runtime.GetHostsByRole(common.Registry)[0].GetArch()
packageDir := filepath.Join(runtime.GetWorkDir(), "registry", arch)
if err := util.CreateDir(packageDir); err != nil {
return errors.Wrap(err, "Failed to create download target dir")
}
if err := RegistryPackageDownloadHTTP(k.KubeConf, packageDir, arch, k.PipelineCache); err != nil {
return err
}
return nil
}

5
pkg/bootstrap/os/templates/init_script.go
View File

@ -18,6 +18,7 @@ package templates
import (
"fmt"
"github.com/kubesphere/kubekey/pkg/bootstrap/registry"
"text/template"
"github.com/kubesphere/kubekey/pkg/common"
@ -167,6 +168,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
}
}
if len(runtime.GetHostsByRole(common.Registry)) > 0 {
hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), registry.RegistryCertificateBaseName))
}
hostsList = append(hostsList, lbHost)
return hostsList
}

142
pkg/bootstrap/registry/certs.go Normal file
View File

@ -0,0 +1,142 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"crypto/x509"
"fmt"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/connector"
"github.com/kubesphere/kubekey/pkg/utils/certs"
"github.com/pkg/errors"
"k8s.io/client-go/util/cert"
certutil "k8s.io/client-go/util/cert"
netutils "k8s.io/utils/net"
"net"
"path/filepath"
"strings"
)
const (
RegistryCertificateBaseName = "dockerhub.kubekey.local"
LocalCertsDir = "localCertsDir"
CertsFileList = "certsFileList"
)
// KubekeyCertEtcdCA is the definition of the root CA used by the hosted etcd server.
func KubekeyCertRegistryCA() *certs.KubekeyCert {
return &certs.KubekeyCert{
Name: "registry-ca",
LongName: "self-signed CA to provision identities for registry",
BaseName: "ca",
Config: certs.CertConfig{
Config: certutil.Config{
CommonName: "registry-ca",
},
},
}
}
// KubekeyCertEtcdAdmin is the definition of the cert for etcd admin.
func KubekeyCertRegistryServer(altNames *certutil.AltNames) *certs.KubekeyCert {
return &certs.KubekeyCert{
Name: "registry-server",
LongName: "certificate for registry server",
BaseName: RegistryCertificateBaseName,
CAName: "registry-ca",
Config: certs.CertConfig{
Config: certutil.Config{
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
AltNames: *altNames,
CommonName: RegistryCertificateBaseName,
},
},
}
}
type FetchCerts struct {
common.KubeAction
}
func (f *FetchCerts) Execute(runtime connector.Runtime) error {
src := "/etc/ssl/registry/ssl"
dst := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir())
certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/registry/ssl/ | grep .pem", false)
if err != nil {
return nil
}
certsList := strings.Split(certs, "\r\n")
if len(certsList) > 0 {
for _, cert := range certsList {
if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil {
return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert)))
}
}
}
return nil
}
type GenerateCerts struct {
common.KubeAction
}
func (g *GenerateCerts) Execute(runtime connector.Runtime) error {
var pkiPath string
if g.KubeConf.Arg.CertificatesDir == "" {
pkiPath = fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir())
}
var altName cert.AltNames
dnsList := []string{"localhost", RegistryCertificateBaseName, runtime.GetHostsByRole(common.Registry)[0].GetName()}
ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback, netutils.ParseIPSloppy(runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress())}
altName.DNSNames = dnsList
altName.IPs = ipList
files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", RegistryCertificateBaseName), fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName)}
// CA
certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()}
// Certs
certsList = append(certsList, KubekeyCertRegistryServer(&altName))
var lastCACert *certs.KubekeyCert
for _, c := range certsList {
if c.CAName == "" {
err := certs.GenerateCA(c, pkiPath, g.KubeConf)
if err != nil {
return err
}
lastCACert = c
} else {
err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf)
if err != nil {
return err
}
}
}
g.ModuleCache.Set(LocalCertsDir, pkiPath)
g.ModuleCache.Set(CertsFileList, files)
return nil
}

311
pkg/bootstrap/registry/module.go Normal file
View File

@ -0,0 +1,311 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"fmt"
"github.com/kubesphere/kubekey/pkg/bootstrap/registry/templates"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/container"
docker_template "github.com/kubesphere/kubekey/pkg/container/templates"
"github.com/kubesphere/kubekey/pkg/core/action"
"github.com/kubesphere/kubekey/pkg/core/prepare"
"github.com/kubesphere/kubekey/pkg/core/task"
"github.com/kubesphere/kubekey/pkg/core/util"
"path/filepath"
)
type RegistryCertsModule struct {
common.KubeModule
Skip bool
}
func (p *RegistryCertsModule) IsSkip() bool {
return p.Skip
}
func (i *RegistryCertsModule) Init() {
i.Name = "InitRegistryModule"
i.Desc = "Init a local registry"
fetchCerts := &task.RemoteTask{
Name: "FetchRegistryCerts",
Desc: "Fetcd registry certs",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: new(FirstRegistryNode),
Action: new(FetchCerts),
Parallel: false,
}
generateCerts := &task.LocalTask{
Name: "GenerateRegistryCerts",
Desc: "Generate registry Certs",
Action: new(GenerateCerts),
}
syncCertsFile := &task.RemoteTask{
Name: "SyncCertsFile",
Desc: "Synchronize certs file",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(SyncCertsFile),
Parallel: true,
Retry: 1,
}
syncCertsToAllNodes := &task.RemoteTask{
Name: "SyncCertsFileToAllNodes",
Desc: "Synchronize certs file to all nodes",
Hosts: i.Runtime.GetHostsByRole(common.K8s),
Action: new(SyncCertsToAllNodes),
Parallel: true,
Retry: 1,
}
i.Tasks = []task.Interface{
fetchCerts,
generateCerts,
syncCertsFile,
syncCertsToAllNodes,
}
}
type InstallRegistryModule struct {
common.KubeModule
}
func (i *InstallRegistryModule) Init() {
i.Name = "InstallRegistryModule"
i.Desc = "Install local registry"
switch i.KubeConf.Cluster.Registry.Type {
case common.Harbor:
i.Tasks = InstallHarbor(i)
default:
i.Tasks = InstallRegistry(i)
}
}
func InstallRegistry(i *InstallRegistryModule) []task.Interface {
installRegistryBinary := &task.RemoteTask{
Name: "InstallRegistryBinary",
Desc: "Install local registry",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(InstallRegistryBinary),
Parallel: true,
Retry: 1,
}
generateRegistryService := &task.RemoteTask{
Name: "GenerateRegistryService",
Desc: "Generate registry service",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: &action.Template{
Template: templates.RegistryServiceTempl,
Dst: "/etc/systemd/system/registry.service",
},
Parallel: true,
Retry: 1,
}
generateRegistryConfig := &task.RemoteTask{
Name: "GenerateRegistryConfig",
Desc: "Generate registry config",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: &action.Template{
Template: templates.RegistryConfigTempl,
Dst: "/etc/kubekey/registry/config.yaml",
Data: util.Data{
"Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName),
"Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName),
},
},
Parallel: true,
Retry: 1,
}
startRgistryService := &task.RemoteTask{
Name: "StartRegistryService",
Desc: "Start registry service",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(StartRegistryService),
Parallel: true,
Retry: 1,
}
return []task.Interface{
installRegistryBinary,
generateRegistryService,
generateRegistryConfig,
startRgistryService,
}
}
func InstallHarbor(i *InstallRegistryModule) []task.Interface {
// Install docker
syncBinaries := &task.RemoteTask{
Name: "SyncDockerBinaries",
Desc: "Sync docker binaries",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: new(container.SyncDockerBinaries),
Parallel: true,
Retry: 2,
}
generateContainerdService := &task.RemoteTask{
Name: "GenerateContainerdService",
Desc: "Generate containerd service",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: &action.Template{
Template: docker_template.ContainerdService,
Dst: filepath.Join("/etc/systemd/system", docker_template.ContainerdService.Name()),
},
Parallel: true,
}
enableContainerd := &task.RemoteTask{
Name: "EnableContainerd",
Desc: "Enable containerd",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: new(container.EnableContainerd),
Parallel: true,
}
generateDockerService := &task.RemoteTask{
Name: "GenerateDockerService",
Desc: "Generate docker service",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: &action.Template{
Template: docker_template.DockerService,
Dst: filepath.Join("/etc/systemd/system", docker_template.DockerService.Name()),
},
Parallel: true,
}
generateDockerConfig := &task.RemoteTask{
Name: "GenerateDockerConfig",
Desc: "Generate docker config",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: &action.Template{
Template: docker_template.DockerConfig,
Dst: filepath.Join("/etc/docker/", docker_template.DockerConfig.Name()),
Data: util.Data{
"Mirrors": docker_template.Mirrors(i.KubeConf),
"InsecureRegistries": docker_template.InsecureRegistries(i.KubeConf),
},
},
Parallel: true,
}
enableDocker := &task.RemoteTask{
Name: "EnableDocker",
Desc: "Enable docker",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{Not: true},
},
Action: new(container.EnableDocker),
Parallel: true,
}
dockerLoginRegistry := &task.RemoteTask{
Name: "Login PrivateRegistry",
Desc: "Add auths to container runtime",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Prepare: &prepare.PrepareCollection{
&container.DockerExist{},
&container.PrivateRegistryAuth{},
},
Action: new(container.DockerLoginRegistry),
Parallel: true,
}
// Install docker compose
installDockerCompose := &task.RemoteTask{
Name: "InstallDockerCompose",
Desc: "Install docker compose",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(InstallDockerCompose),
Parallel: true,
Retry: 2,
}
// Install Harbor
syncHarborPackage := &task.RemoteTask{
Name: "SyncHarborPackage",
Desc: "Sync harbor package",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(SyncHarborPackage),
Parallel: true,
Retry: 2,
}
generateHarborConfig := &task.RemoteTask{
Name: "GenerateHarborConfig",
Desc: "Generate harbor config",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: &action.Template{
Template: templates.HarborConfigTempl,
Dst: "/opt/harbor/harbor.yml",
Data: util.Data{
"Domain": RegistryCertificateBaseName,
"Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName),
"Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName),
},
},
Parallel: true,
Retry: 1,
}
startHarbor := &task.RemoteTask{
Name: "StartHarbor",
Desc: "start harbor",
Hosts: i.Runtime.GetHostsByRole(common.Registry),
Action: new(StartHarbor),
Parallel: true,
Retry: 2,
}
return []task.Interface{
syncBinaries,
generateContainerdService,
enableContainerd,
generateDockerService,
generateDockerConfig,
enableDocker,
dockerLoginRegistry,
installDockerCompose,
syncHarborPackage,
generateHarborConfig,
startHarbor,
}
}

34
pkg/bootstrap/registry/prepares.go Normal file
View File

@ -0,0 +1,34 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/connector"
)
type FirstRegistryNode struct {
common.KubePrepare
Not bool
}
func (f *FirstRegistryNode) PreCheck(runtime connector.Runtime) (bool, error) {
if runtime.GetHostsByRole(common.Registry)[0].GetName() == runtime.RemoteHost().GetName() {
return !f.Not, nil
}
return f.Not, nil
}

209
pkg/bootstrap/registry/tasks.go Normal file
View File

@ -0,0 +1,209 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"fmt"
kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/connector"
"github.com/kubesphere/kubekey/pkg/core/logger"
"github.com/kubesphere/kubekey/pkg/utils"
"github.com/pkg/errors"
"path/filepath"
"strings"
)
type SyncCertsFile struct {
common.KubeAction
}
func (s *SyncCertsFile) Execute(runtime connector.Runtime) error {
localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir)
if !ok {
return errors.New("get etcd local certs dir by module cache failed")
}
files, ok := s.ModuleCache.Get(CertsFileList)
if !ok {
return errors.New("get etcd certs file list by module cache failed")
}
dir := localCertsDir.(string)
fileList := files.([]string)
for _, fileName := range fileList {
if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, fileName)); err != nil {
return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed")
}
}
return nil
}
type SyncCertsToAllNodes struct {
common.KubeAction
}
func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error {
localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir)
if !ok {
return errors.New("get etcd local certs dir by module cache failed")
}
files, ok := s.ModuleCache.Get(CertsFileList)
if !ok {
return errors.New("get etcd certs file list by module cache failed")
}
dir := localCertsDir.(string)
fileList := files.([]string)
var dstDir string
switch s.KubeConf.Cluster.Kubernetes.ContainerManager {
case common.Docker:
dstDir = fmt.Sprintf("/etc/docker/certs.d/%s", RegistryCertificateBaseName)
case common.Conatinerd:
dstDir = common.RegistryCertDir
case common.Crio:
// TODO: Add the steps of cri-o's installation.
case common.Isula:
// TODO: Add the steps of iSula's installation.
default:
logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(s.KubeConf.Cluster.Kubernetes.ContainerManager))
}
for _, fileName := range fileList {
var dstFileName string
switch fileName {
case "ca.pem":
dstFileName = "ca.crt"
case "ca-key.pem":
continue
default:
if strings.HasSuffix(fileName, "-key.pem") {
dstFileName = strings.Replace(fileName, "-key.pem", ".key", -1)
} else {
dstFileName = strings.Replace(fileName, ".pem", ".cert", -1)
}
}
if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(dstDir, dstFileName)); err != nil {
return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed")
}
}
return nil
}
type InstallRegistryBinary struct {
common.KubeAction
}
func (g *InstallRegistryBinary) Execute(runtime connector.Runtime) error {
if err := utils.ResetTmpDir(runtime); err != nil {
return err
}
registryFile := fmt.Sprintf("registry-2-linux-%s", runtime.RemoteHost().GetArch())
filesDir := filepath.Join(runtime.GetWorkDir(), "registry", runtime.RemoteHost().GetArch())
if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s.tar.gz", filesDir, registryFile), fmt.Sprintf("%s/%s.tar.gz", common.TmpDir, registryFile)); err != nil {
return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed")
}
installCmd := fmt.Sprintf("tar -zxf %s/%s.tar.gz && mv -f registry /usr/local/bin/ && chmod +x /usr/local/bin/registry", common.TmpDir, registryFile)
if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil {
return errors.Wrap(errors.WithStack(err), "install etcd binaries failed")
}
return nil
}
type StartRegistryService struct {
common.KubeAction
}
func (g *StartRegistryService) Execute(runtime connector.Runtime) error {
installCmd := "systemctl daemon-reload && systemctl enable registry && systemctl restart registry"
if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil {
return errors.Wrap(errors.WithStack(err), "start registry service failed")
}
fmt.Println()
fmt.Println("Local image registry created successfully. Address: dockerhub.kubekey.local")
fmt.Println()
return nil
}
type InstallDockerCompose struct {
common.KubeAction
}
func (g *InstallDockerCompose) Execute(runtime connector.Runtime) error {
if err := utils.ResetTmpDir(runtime); err != nil {
return err
}
dockerComposeFile := "docker-compose-linux-x86_64"
filesDir := filepath.Join(runtime.GetWorkDir(), runtime.RemoteHost().GetArch())
if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s", filesDir, dockerComposeFile), fmt.Sprintf("%s/%s", common.TmpDir, dockerComposeFile)); err != nil {
return errors.Wrap(errors.WithStack(err), "sync docker-compose failed")
}
installCmd := fmt.Sprintf("mv -f %s/%s /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose", common.TmpDir, dockerComposeFile)
if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil {
return errors.Wrap(errors.WithStack(err), "install dokcer-compose failed")
}
return nil
}
type SyncHarborPackage struct {
common.KubeAction
}
func (g *SyncHarborPackage) Execute(runtime connector.Runtime) error {
if err := utils.ResetTmpDir(runtime); err != nil {
return err
}
harborPackageFile := fmt.Sprintf("harbor-offline-installer-%s.tgz", kubekeyapiv1alpha2.DefaultHarborVersion)
filesDir := filepath.Join(runtime.GetWorkDir(), "registry", runtime.RemoteHost().GetArch())
if err := runtime.GetRunner().Scp(fmt.Sprintf("%s/%s", filesDir, harborPackageFile), fmt.Sprintf("%s/%s", common.TmpDir, harborPackageFile)); err != nil {
return errors.Wrap(errors.WithStack(err), "sync harbor package failed")
}
installCmd := fmt.Sprintf("tar -zxvf %s/%s -C /opt", common.TmpDir, harborPackageFile)
if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil {
return errors.Wrap(errors.WithStack(err), "unzip harbor package failed")
}
return nil
}
type StartHarbor struct {
common.KubeAction
}
func (g *StartHarbor) Execute(runtime connector.Runtime) error {
startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-notary --with-trivy --with-chartmuseum"
if _, err := runtime.GetRunner().SudoCmd(startCmd, false); err != nil {
return errors.Wrap(errors.WithStack(err), "start harbor failed")
}
fmt.Println()
fmt.Println("Local image registry created successfully. Address: dockerhub.kubekey.local")
fmt.Println()
return nil
}

119
pkg/bootstrap/registry/templates/harbor.go Normal file
View File

@ -0,0 +1,119 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package templates
import (
"github.com/lithammer/dedent"
"text/template"
)
var (
// HarborConfigTempl defines the template of registry's configuration file.
HarborConfigTempl = template.Must(template.New("harborConfig").Parse(
dedent.Dedent(`# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: {{ .Domain }}
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /etc/ssl/registry/ssl/{{ .Certificate }}
private_key: /etc/ssl/registry/ssl/{{ .Key }}
# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: Harbor12345
# Harbor DB configuration
database:
# The password for the root user of Harbor DB. Change this before any production use.
password: root123
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
max_idle_conns: 100
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
# Note: the default number of connections is 1024 for postgres of harbor.
max_open_conns: 900
# The default data volume
data_volume: /mnt/registry
# Trivy configuration
#
# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached
# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it
# should download a newer version from the Internet or use the cached one. Currently, the database is updated every
# 12 hours and published as a new release to GitHub.
trivy:
# ignoreUnfixed The flag to display only fixed vulnerabilities
ignore_unfixed: false
# skipUpdate The flag to enable or disable Trivy DB downloads from GitHub
#
skip_update: false
#
# insecure The flag to skip verifying registry certificate
insecure: false
jobservice:
# Maximum number of job workers in job service
max_job_workers: 10
notification:
# Maximum retry count for webhook job
webhook_job_max_retry: 10
chart:
# Change the value of absolute_url to enabled can enable absolute url in chart
absolute_url: disabled
# Log configurations
log:
# options are debug, info, warning, error, fatal
level: info
# configs for logs in local storage
local:
# Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
rotate_count: 50
# Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
# If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
# are all valid.
rotate_size: 200M
# The directory on your host that store log
location: /var/log/harbor
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version: 2.4.0
# Global proxy
proxy:
http_proxy:
https_proxy:
no_proxy:
components:
- core
- jobservice
- trivy
`)))
)

52
pkg/bootstrap/registry/templates/registry.go Normal file
View File

@ -0,0 +1,52 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package templates
import (
"github.com/lithammer/dedent"
"text/template"
)
var (
// RegistryServiceTempl defines the template of registry service for systemd.
RegistryServiceTempl = template.Must(template.New("registryService").Parse(
dedent.Dedent(`[Unit]
Description=v2 Registry server for Container
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/bin/registry serve /etc/kubekey/registry/config.yaml
Restart=on-failure
[Install]
WantedBy=multi-user.target
`)))
// RegistryConfigTempl defines the template of registry's configuration file.
RegistryConfigTempl = template.Must(template.New("registryConfig").Parse(
dedent.Dedent(`version: 0.1
log:
fields:
service: registry
storage:
cache:
layerinfo: inmemory
filesystem:
rootdirectory: /mnt/registry
http:
addr: :443
tls:
certificate: /etc/ssl/registry/ssl/{{ .Certificate }}
key: /etc/ssl/registry/ssl/{{ .Key }}
`)))
)

15
pkg/common/common.go
View File

@ -26,11 +26,13 @@ const (
File = "file"
Operator = "operator"
Master = "master"
Worker = "worker"
ETCD = "etcd"
K8s = "k8s"
KubeKey = "kubekey"
Master = "master"
Worker = "worker"
ETCD = "etcd"
K8s = "k8s"
Registry = "registry"
KubeKey = "kubekey"
Harbor = "harbor"
KubeBinaries = "KubeBinaries"
@ -43,7 +45,8 @@ const (
KubeScriptDir = "/usr/local/bin/kube-scripts"
KubeletFlexvolumesPluginsDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
ETCDCertDir = "/etc/ssl/etcd/ssl"
ETCDCertDir = "/etc/ssl/etcd/ssl"
RegistryCertDir = "/etc/ssl/registry/ssl"
HaproxyDir = "/etc/kubekey/haproxy"

3
pkg/common/kube_runtime.go
View File

@ -85,6 +85,9 @@ func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) {
if v.IsMaster || v.IsWorker {
host.SetRole(K8s)
}
if v.IsRegistry {
host.SetRole(Registry)
}
base.AppendHost(host)
base.AppendRoleMap(host)
}

7
pkg/common/loader.go
View File

@ -101,9 +101,10 @@ func (d *DefaultLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) {
})
allInOne.Spec.RoleGroups = kubekeyapiv1alpha2.RoleGroups{
Etcd: []string{hostname},
Master: []string{hostname},
Worker: []string{hostname},
Etcd: []string{hostname},
Master: []string{hostname},
Worker: []string{hostname},
Registry: []string{hostname},
}
if d.KubernetesVersion != "" {
s := strings.Split(d.KubernetesVersion, "-")

1
pkg/config/templates/cluster.go
View File

@ -59,6 +59,7 @@ spec:
# multus support. https://github.com/k8snetworkplumbingwg/multus-cni
enableMultusCNI: false
registry:
privateRegistry: ""
registryMirrors: []
insecureRegistries: []
addons: []

2
pkg/container/containerd.go
View File

@ -36,7 +36,7 @@ func (s *SyncCrictlBinaries) Execute(runtime connector.Runtime) error {
return err
}
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries)
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
if !ok {
return errors.New("get KubeBinary by pipeline cache failed")
}

2
pkg/container/docker.go
View File

@ -38,7 +38,7 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error {
return err
}
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries)
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
if !ok {
return errors.New("get KubeBinary by pipeline cache failed")
}

1
pkg/container/templates/containerd_config.go
View File

@ -84,5 +84,4 @@ state = "/run/containerd"
password = "{{$entry.Password}}"
{{- end}}
{{- end}}
`)))

2
pkg/etcd/certs.go
View File

@ -82,7 +82,7 @@ func KubekeyCertEtcdMember(hostname string, altNames *certutil.AltNames) *certs.
}
}
// KubekeyCertEtcdMember is the definition of the cert for etcd client.
// KubekeyCertEtcdClient is the definition of the cert for etcd client.
func KubekeyCertEtcdClient(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert {
l := strings.Split(hostname, ".")
return &certs.KubekeyCert{

40
pkg/files/file.go
View File

@ -23,17 +23,20 @@ import (
)
const (
kubeadm = "kubeadm"
kubelet = "kubelet"
kubectl = "kubectl"
kubecni = "kubecni"
etcd = "etcd"
helm = "helm"
amd64 = "amd64"
arm64 = "arm64"
k3s = "k3s"
docker = "docker"
crictl = "crictl"
kubeadm = "kubeadm"
kubelet = "kubelet"
kubectl = "kubectl"
kubecni = "kubecni"
etcd = "etcd"
helm = "helm"
amd64 = "amd64"
arm64 = "arm64"
k3s = "k3s"
docker = "docker"
crictl = "crictl"
registry = "registry"
harbor = "harbor"
compose = "compose"
)
type KubeBinary struct {
@ -362,6 +365,21 @@ var (
"v1.22.0": "a713c37fade0d96a989bc15ebe906e08ef5c8fe5e107c2161b0665e9963b770e",
},
},
registry: {
amd64: {
"2": "7706e46674fa2cf20f734dfb7e4dd7f1390710e9c0a2c520563e3c55f3e4b5c5",
},
},
compose: {
amd64: {
"v2.2.2": "92551cd3d22b41536ce8345fe06795ad0d08cb3c17b693ecbfe41176e501bfd4",
},
},
harbor: {
amd64: {
"v2.4.1": "cfd799c150b59353aefb34835f3a2e859763cb2e91966cd3ffeb1b6ceaa19841",
},
},
}
)

2
pkg/k3s/tasks.go
View File

@ -88,7 +88,7 @@ type SyncKubeBinary struct {
}
func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error {
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries)
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
if !ok {
return errors.New("get KubeBinary by pipeline cache failed")
}

2
pkg/kubernetes/tasks.go
View File

@ -101,7 +101,7 @@ type SyncKubeBinary struct {
}
func (i *SyncKubeBinary) Execute(runtime connector.Runtime) error {
binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries)
binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
if !ok {
return errors.New("get KubeBinary by pipeline cache failed")
}

2
pkg/kubesphere/modules.go
View File

@ -111,6 +111,8 @@ func (d *DeployModule) Init() {
d.Tasks = []task.Interface{
generateManifests,
// apply crd installer.kubesphere.io/v1alpha1
apply,
addConfig,
createNamespace,
setup,

2
pkg/pipelines/add_nodes.go
View File

@ -23,6 +23,7 @@ import (
"github.com/kubesphere/kubekey/pkg/bootstrap/confirm"
"github.com/kubesphere/kubekey/pkg/bootstrap/os"
"github.com/kubesphere/kubekey/pkg/bootstrap/precheck"
"github.com/kubesphere/kubekey/pkg/bootstrap/registry"
"github.com/kubesphere/kubekey/pkg/certs"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/container"
@ -43,6 +44,7 @@ func NewAddNodesPipeline(runtime *common.KubeRuntime) error {
&confirm.InstallConfirmModule{Skip: runtime.Arg.SkipConfirmCheck},
&binaries.NodeBinariesModule{},
&os.ConfigureOSModule{},
&registry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0},
&kubernetes.StatusModule{},
&container.InstallContainerModule{},
&images.PullModule{Skip: runtime.Arg.SkipPullImages},

72
pkg/pipelines/init_registry.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pipelines
import (
"fmt"
"github.com/kubesphere/kubekey/pkg/binaries"
"github.com/kubesphere/kubekey/pkg/bootstrap/os"
"github.com/kubesphere/kubekey/pkg/bootstrap/registry"
"github.com/kubesphere/kubekey/pkg/common"
"github.com/kubesphere/kubekey/pkg/core/module"
"github.com/kubesphere/kubekey/pkg/core/pipeline"
)
func NewInitRegistryPipeline(runtime *common.KubeRuntime) error {
m := []module.Module{
&binaries.RegistryPackageModule{},
&os.ConfigureOSModule{},
&registry.RegistryCertsModule{},
&registry.InstallRegistryModule{},
}
p := pipeline.Pipeline{
Name: "InitRegistryPipeline",
Modules: m,
Runtime: runtime,
}
if err := p.Start(); err != nil {
return err
}
return nil
}
func InitRegistry(args common.Argument, downloadCmd string) error {
args.DownloadCommand = func(path, url string) string {
// this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under
// some poor network environment. Or users even can choose another cli, it might be wget.
// perhaps we should have a build-in download function instead of totally rely on the external one
return fmt.Sprintf(downloadCmd, path, url)
}
var loaderType string
if args.FilePath != "" {
loaderType = common.File
} else {
loaderType = common.AllInOne
}
runtime, err := common.NewKubeRuntime(loaderType, args)
if err != nil {
return err
}
if err := NewInitRegistryPipeline(runtime); err != nil {
return err
}
return nil
}