From d81ea21da794779d005bf73f6649ef4254f9828a Mon Sep 17 00:00:00 2001 From: 24sama Date: Thu, 20 Jan 2022 11:13:53 +0800 Subject: [PATCH] Support the ArtifactModule for authorized pull images Signed-off-by: 24sama --- apis/kubekey/v1alpha2/cluster_types.go | 2 +- apis/kubekey/v1alpha2/manifest_types.go | 6 ++++ .../kubekey/v1alpha2/zz_generated.deepcopy.go | 17 +++++++++ .../kubekey.kubesphere.io_manifests.yaml | 6 ++++ pkg/artifact/containerd.go | 35 ++++++++++++++++++- pkg/artifact/tasks.go | 19 ++++++++-- pkg/artifact/templates/manifest.go | 16 +++++---- pkg/common/artifact_runtime.go | 12 +++++-- pkg/images/images.go | 3 +- 9 files changed, 100 insertions(+), 16 deletions(-) diff --git a/apis/kubekey/v1alpha2/cluster_types.go b/apis/kubekey/v1alpha2/cluster_types.go index 68ba8932..1e0d21bd 100644 --- a/apis/kubekey/v1alpha2/cluster_types.go +++ b/apis/kubekey/v1alpha2/cluster_types.go @@ -174,7 +174,7 @@ type RegistryConfig struct { InsecureRegistries []string `yaml:"insecureRegistries" json:"insecureRegistries,omitempty"` PrivateRegistry string `yaml:"privateRegistry" json:"privateRegistry,omitempty"` PlainHTTP bool `yaml:"plainHTTP" json:"plainHTTP,omitempty"` - Auths runtime.RawExtension `yaml:"Auths" json:"Auths,omitempty"` + Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` } // KubeSphere defines the configuration information of the KubeSphere. diff --git a/apis/kubekey/v1alpha2/manifest_types.go b/apis/kubekey/v1alpha2/manifest_types.go index 453009f1..f4ca6923 100644 --- a/apis/kubekey/v1alpha2/manifest_types.go +++ b/apis/kubekey/v1alpha2/manifest_types.go @@ -18,6 +18,7 @@ package v1alpha2 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -94,6 +95,10 @@ type Components struct { DockerCompose DockerCompose `yaml:"docker-compose" json:"docker-compose"` } +type ManifestRegistry struct { + Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` +} + // ManifestSpec defines the desired state of Manifest type ManifestSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -104,6 +109,7 @@ type ManifestSpec struct { KubernetesDistributions []KubernetesDistribution `yaml:"kubernetesDistributions" json:"kubernetesDistributions"` Components Components `yaml:"components" json:"components"` Images []string `yaml:"images" json:"images"` + ManifestRegistry ManifestRegistry `yaml:"registry" json:"registry"` } // ManifestStatus defines the observed state of Manifest diff --git a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go index d725d44c..d6bc3662 100644 --- a/apis/kubekey/v1alpha2/zz_generated.deepcopy.go +++ b/apis/kubekey/v1alpha2/zz_generated.deepcopy.go @@ -704,6 +704,22 @@ func (in *ManifestList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManifestRegistry) DeepCopyInto(out *ManifestRegistry) { + *out = *in + in.Auths.DeepCopyInto(&out.Auths) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManifestRegistry. +func (in *ManifestRegistry) DeepCopy() *ManifestRegistry { + if in == nil { + return nil + } + out := new(ManifestRegistry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManifestSpec) DeepCopyInto(out *ManifestSpec) { *out = *in @@ -728,6 +744,7 @@ func (in *ManifestSpec) DeepCopyInto(out *ManifestSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + in.ManifestRegistry.DeepCopyInto(&out.ManifestRegistry) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManifestSpec. diff --git a/config/crd/bases/kubekey.kubesphere.io_manifests.yaml b/config/crd/bases/kubekey.kubesphere.io_manifests.yaml index 6bebaa39..abb8390a 100644 --- a/config/crd/bases/kubekey.kubesphere.io_manifests.yaml +++ b/config/crd/bases/kubekey.kubesphere.io_manifests.yaml @@ -164,12 +164,18 @@ spec: - version type: object type: array + registry: + properties: + auths: + type: object + type: object required: - arches - components - images - kubernetesDistributions - operationSystems + - registry type: object status: description: ManifestStatus defines the observed state of Manifest diff --git a/pkg/artifact/containerd.go b/pkg/artifact/containerd.go index 4592f207..be28f877 100644 --- a/pkg/artifact/containerd.go +++ b/pkg/artifact/containerd.go @@ -19,20 +19,33 @@ package artifact import ( "context" "crypto/tls" + "encoding/json" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/config" + "github.com/kubesphere/kubekey/pkg/common" + "github.com/kubesphere/kubekey/pkg/core/logger" ) // PushTracker returns a new InMemoryTracker which tracks the ref status var PushTracker = docker.NewInMemoryTracker() -func GetResolver(ctx context.Context) remotes.Resolver { +func GetResolver(ctx context.Context, auth registryAuth) remotes.Resolver { + username := auth.Username + secret := auth.Password + options := docker.ResolverOptions{ Tracker: PushTracker, } hostOptions := config.HostOptions{} + hostOptions.Credentials = func(host string) (string, string, error) { + return username, secret, nil + } + + if auth.PlainHTTP { + hostOptions.DefaultScheme = "http" + } defaultConfig := &tls.Config{ InsecureSkipVerify: true, @@ -43,3 +56,23 @@ func GetResolver(ctx context.Context) remotes.Resolver { options.Hosts = config.ConfigureHosts(ctx, hostOptions) return docker.NewResolver(options) } + +type registryAuth struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + PlainHTTP bool `json:"plainHTTP,omitempty"` +} + +func Auths(manifest *common.ArtifactManifest) (auths map[string]registryAuth) { + if len(manifest.Spec.ManifestRegistry.Auths.Raw) == 0 { + return + } + + err := json.Unmarshal(manifest.Spec.ManifestRegistry.Auths.Raw, &auths) + if err != nil { + logger.Log.Fatal("Failed to Parse Registry Auths configuration: %v", manifest.Spec.ManifestRegistry.Auths.Raw) + return + } + + return +} diff --git a/pkg/artifact/tasks.go b/pkg/artifact/tasks.go index 2931ea63..ae6dd9a3 100644 --- a/pkg/artifact/tasks.go +++ b/pkg/artifact/tasks.go @@ -25,6 +25,7 @@ import ( "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes" "github.com/kubesphere/kubekey/pkg/common" "github.com/kubesphere/kubekey/pkg/core/connector" "github.com/kubesphere/kubekey/pkg/core/logger" @@ -62,9 +63,17 @@ func (p *PullImages) Execute(_ connector.Runtime) error { if !ok { return errors.New("get containerd client failed by pipeline client") } - client := c.(*containerd.Client) + client := c.(*containerd.Client) ctx := namespaces.WithNamespace(context.Background(), "kubekey") + + // k: registry name "registry-1.docker.io", v: registryAuth + auths := Auths(p.Manifest) + resolvers := make(map[string]remotes.Resolver) + for repo, auth := range auths { + resolvers[repo] = GetResolver(ctx, auth) + } + for _, image := range p.Manifest.Spec.Images { progress := make(chan struct{}) lctx, done, err := client.WithLease(ctx) @@ -89,11 +98,16 @@ func (p *PullImages) Execute(_ connector.Runtime) error { }) opts := []containerd.RemoteOpt{ - containerd.WithResolver(GetResolver(ctx)), containerd.WithImageHandler(h), containerd.WithSchema1Conversion, } + if resolver, ok := resolvers[strings.Split(image, "/")[0]]; ok { + opts = append(opts, containerd.WithResolver(resolver)) + } else { + opts = append(opts, containerd.WithResolver(GetResolver(ctx, registryAuth{}))) + } + for _, arch := range p.Manifest.Spec.Arches { opts = append(opts, containerd.WithPlatform(arch)) } @@ -106,7 +120,6 @@ func (p *PullImages) Execute(_ connector.Runtime) error { } <-progress - //todo: Whether it need to be unpack? } return nil diff --git a/pkg/artifact/templates/manifest.go b/pkg/artifact/templates/manifest.go index ad0d0fcb..54d69e21 100644 --- a/pkg/artifact/templates/manifest.go +++ b/pkg/artifact/templates/manifest.go @@ -40,7 +40,7 @@ spec: - arch: {{ $v.Arch }} type: {{ $v.Type }} id: {{ $v.Id }} - version: {{ $v.Version }} + version: "{{ $v.Version }}" osImage: {{ $v.OsImage }} repository: iso: @@ -69,16 +69,18 @@ spec: crictl: version: {{ .Options.Components.Crictl.Version }} ## - #docker-registry: - # version: 2 - #harbor: - # version: v2.4.1 - #docker-compose: - # version: v2.2.2 + # docker-registry: + # version: "2" + # harbor: + # version: v2.4.1 + # docker-compose: + # version: v2.2.2 images: {{- range .Options.Images }} - {{ . }} {{- end }} + registry: + auths: {} `))) diff --git a/pkg/common/artifact_runtime.go b/pkg/common/artifact_runtime.go index cbe5deca..951e99e8 100644 --- a/pkg/common/artifact_runtime.go +++ b/pkg/common/artifact_runtime.go @@ -17,11 +17,12 @@ package common import ( + "encoding/json" kubekeyv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2" "github.com/kubesphere/kubekey/pkg/core/connector" "github.com/pkg/errors" - "gopkg.in/yaml.v2" "io/ioutil" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" "path/filepath" ) @@ -56,9 +57,14 @@ func NewArtifactRuntime(arg ArtifactArgument) (*ArtifactRuntime, error) { return nil, errors.Wrapf(err, "Failed to read file %s", fp) } + contentToJson, err := k8syaml.ToJSON(fileByte) + if err != nil { + return nil, errors.Wrap(err, "Unable to convert configuration to json") + } + manifest := &kubekeyv1alpha2.Manifest{} - if err := yaml.Unmarshal(fileByte, manifest); err != nil { - return nil, errors.Wrapf(err, "Failed to yaml unmarshal") + if err := json.Unmarshal(contentToJson, manifest); err != nil { + return nil, errors.Wrapf(err, "Failed to json unmarshal") } r := &ArtifactRuntime{ diff --git a/pkg/images/images.go b/pkg/images/images.go index 1fe0704b..b0f7a4c6 100644 --- a/pkg/images/images.go +++ b/pkg/images/images.go @@ -175,10 +175,11 @@ func CmdPush(fileName string, prePath string, kubeConf *common.KubeConf, arches pushCmd = fmt.Sprintf("%s --user %s:%s", pushCmd, auth.Username, auth.Password) } + logger.Log.Debugf("push command: %s", pushCmd) if out, err := exec.Command("/bin/bash", "-c", pushCmd).CombinedOutput(); err != nil { return errors.Wrapf(err, "push image %s failed: %s", oldName, out) } - logger.Log.Messagef(common.LocalHost, "Push %s success", image.ImageName()) + fmt.Printf("push %s success \n", image.ImageName()) return nil }