From 867aca2b7dfea131100fff7577d2cfedf101ed26 Mon Sep 17 00:00:00 2001 From: zuoxuesong-worker Date: Fri, 28 Nov 2025 13:32:00 +0800 Subject: [PATCH] feat: image module add copy func (#2868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: update some default config values (#2866) feat: update some default config values Signed-off-by: xuesongzuo@yunify.com bugfix: fix artifact image tag set func (#2870) Signed-off-by: xuesongzuo@yunify.com feat: k8s add haproxy image default value (#2869) Signed-off-by: xuesongzuo@yunify.com feat: kk 4.0 制品导出 支持skip_tls_verify 私仓镜像 #2854 (#2855) * feat: kk 4.0 制品导出 支持skip_tls_verify 私仓镜像 #2854 * feat: update image skip tls verify func Signed-off-by: xuesongzuo@yunify.com * feat: update image skip tls verify func Signed-off-by: xuesongzuo@yunify.com * feat: update image skip tls verify func Signed-off-by: xuesongzuo@yunify.com --------- Signed-off-by: xuesongzuo@yunify.com Co-authored-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com * feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com feat: image module add copy func Signed-off-by: xuesongzuo@yunify.com --------- Signed-off-by: xuesongzuo@yunify.com --- docs/zh/modules/image.md | 23 +++++- pkg/modules/image.go | 172 ++++++++++++++++++++++++++++++++++----- 2 files changed, 174 insertions(+), 21 deletions(-) diff --git a/docs/zh/modules/image.md b/docs/zh/modules/image.md index cbef274c..8667a4f2 100644 --- a/docs/zh/modules/image.md +++ b/docs/zh/modules/image.md @@ -26,6 +26,14 @@ image模块允许用户下载镜像到本地目录或上传镜像到远程目录 | push.skip_tls_verify | 默认的是否跳过远程仓库的tls认证 | bool | 否 | - | | push.src_pattern | 正则表达式，过滤本地目录中存放的镜像 | map | 否 | - | | push.dest | 模版语法，从本地目录镜像推送到的远程仓库镜像 | map | 否 | - | +| copy | 模版语法，将镜像在文件系统和镜像仓库内相互复制 | map | 否 | - | +| copy.from | 模版语法，源镜像信息 | map | 否 | - | +| copy.from.path | 镜像源文件路径 | 字符串 | 否 | - | +| copy.from.manifests | 源镜像列表 | 字符串数组 | 否 | - | +| copy.to | 模版语法，从本地目录镜像推送到的远程仓库镜像 | map | 否 | - | +| copy.to.path | 镜像目标文件路径 | 字符串 | 否 | - | +| copy.to.pattern | 正则表达式，过滤复制至目标的镜像 | 字符串 | 否 | - | + 每个本地目录存放的镜像对应一个dest镜像。 ```txt @@ -72,4 +80,17 @@ image模块允许用户下载镜像到本地目录或上传镜像到远程目录 即： docker.io/kubesphere/ks-apiserver:v4.1.3 => hub.kubekey/kubesphere/ks-apiserver:v4.1.3 docker.io/kubesphere/ks-controller-manager:v4.1.3 => hub.kubekey/kubesphere/ks-controller-manager:v4.1.3 -docker.io/kubesphere/ks-console:3.19 => hub.kubekey/kubesphere/ks-console:v4.1.3 \ No newline at end of file +docker.io/kubesphere/ks-console:3.19 => hub.kubekey/kubesphere/ks-console:v4.1.3 + +3. 将镜像从文件系统复制至其他文件系统 +```yaml +- name: file to file + image: + copy: + from: + path: "/tmp/images/" + manifests: + - docker.io/calico/apiserver:v3.28.2 + to: + path: "/tmp/others/images/" +``` \ No newline at end of file diff --git a/pkg/modules/image.go b/pkg/modules/image.go index ab9a3895..aa9a78c7 100644 --- a/pkg/modules/image.go +++ b/pkg/modules/image.go @@ -59,22 +59,37 @@ image: pull: # optional: pull configuration manifests: []string # required: list of image manifests to pull images_dir: string # required: directory to store pulled images - skipTLSVerify: bool # optional: default skip TLS verification + skipTLSVerify: bool # optional: skip TLS verification autus: # optional: target image repo access information, slice type - repo: string # optional: target image repo username: string # optional: target image repo access username password: string # optional: target image repo access password - insecure: bool # optional: skip TLS verification for current repo push: # optional: push configuration - autus: # optional: target image repo access information, slice type - - repo: string # optional: target image repo - username: string # optional: target image repo access username - password: string # optional: target image repo access password - insecure: bool # optional: skip TLS verification for current repo + username: string # optional: registry username + password: string # optional: registry password images_dir: string # required: directory containing images to push - skipTLSVerify: bool # optional: default skip TLS verification + skipTLSVerify: bool # optional: skip TLS verification src_pattern: string # optional: source image pattern to push (regex supported). If not specified, all images in images_dir will be pushed dest: string # required: destination registry and image name. Supports template syntax for dynamic values + copy: + from: + type: string # required: image source type, file or hub + path: string # optional: when image source type is file, then required, means image file path + manifests: []string # required: list of image manifests to pull + skipTLSVerify: bool # optional: skip TLS verification + autus: # optional: target image repo access information, slice type + - repo: string # optional: target image repo + username: string # optional: target image repo access username + password: string # optional: target image repo access password + to: + type: string # required: image target type, file or hub + path: string # required: image target path + skipTLSVerify: bool # optional: skip TLS verification + pattern: string # optional: source image pattern to push (regex supported). If not specified, all images in images_dir will be pushed + autus: # optional: target image repo access information, slice type + - repo: string # optional: target image repo + username: string # optional: target image repo access username + password: string # optional: target image repo access password Usage Examples in Playbook Tasks: 1. Pull images from registry: @@ -101,19 +116,28 @@ Usage Examples in Playbook Tasks: - name: Push images to private registry image: push: - auths: - - repo: docker.io - username: MyDockerAccount - password: my_password - - repo: registry.example.com - username: admin - password: secret + username: admin + password: secret namespace_override: custom-ns images_dir: /path/to/images dest: registry.example.com/{{ . }} register: push_result ``` +3. Copy image from file to file + ```yaml + - name: file to file + image: + copy: + from: + path: "/path/from/images" + manifests: + - nginx:latest + - prometheus:v2.45.0 + to: + path: /path/to/images + ``` + Return Values: - On success: Returns "Success" in stdout - On failure: Returns error message in stderr @@ -125,6 +149,7 @@ const defaultRegistry = "docker.io" type imageArgs struct { pull *imagePullArgs push *imagePushArgs + copy *imageCopyArgs } // imagePullArgs contains parameters for pulling images @@ -155,7 +180,7 @@ func (i imagePullArgs) pull(ctx context.Context, platform string) error { Client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: skipTlsVerifyFunc(img, i.auths, *i.skipTLSVerify), + InsecureSkipVerify: skipTLSVerifyFunc(img, i.auths, *i.skipTLSVerify), }, }, }, @@ -186,6 +211,20 @@ func (i imagePullArgs) pull(ctx context.Context, platform string) error { return nil } +func dockerHostParser(img string) string { + // if image is like docker.io/xxx/xxx:tag, then download by pull func will store it to registry-1.docker.io + // so we should change host from docker.io to registry-1.docker.io + splitedImg := strings.Split(img, "/") + if len(splitedImg) == 1 { + return img + } + if splitedImg[0] != "docker.io" { + return img + } + splitedImg[0] = "registry-1.docker.io" + return strings.Join(splitedImg, "/") +} + func authFunc(auths []imageAuth) func(ctx context.Context, hostport string) (auth.Credential, error) { var creds = make(map[string]auth.Credential) for _, inputAuth := range auths { @@ -207,15 +246,14 @@ func authFunc(auths []imageAuth) func(ctx context.Context, hostport string) (aut } } -func skipTlsVerifyFunc(img string, auths []imageAuth, defaults bool) bool { +func skipTLSVerifyFunc(img string, auths []imageAuth, defaults bool) bool { imgHost := strings.Split(img, "/")[0] for _, a := range auths { if imgHost == a.Repo { if a.Insecure != nil { return *a.Insecure - } else { - return defaults } + return defaults } } return defaults @@ -287,7 +325,7 @@ func (i imagePushArgs) push(ctx context.Context, hostVars map[string]any) error Client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: skipTlsVerifyFunc(dest, i.auths, *i.skipTLSVerify), + InsecureSkipVerify: skipTLSVerifyFunc(dest, i.auths, *i.skipTLSVerify), }, }, }, @@ -303,6 +341,78 @@ func (i imagePushArgs) push(ctx context.Context, hostVars map[string]any) error return nil } +type imageCopyArgs struct { + From imageCopyTargetArgs `json:"from"` + To imageCopyTargetArgs `json:"to"` +} + +type imageCopyTargetArgs struct { + Path string `json:"path"` + manifests []string + Pattern *regexp.Regexp +} + +func (i *imageCopyArgs) parseFromVars(vars, cp map[string]any) error { + i.From.manifests, _ = variable.StringSliceVar(vars, cp, "from", "manifests") + + i.From.Path, _ = variable.StringVar(vars, cp, "from", "path") + + toPath, _ := variable.PrintVar(cp, "to", "path") + if destStr, ok := toPath.(string); !ok { + return errors.New("\"copy.to.path\" must be a string") + } else if destStr == "" { + return errors.New("\"copy.to.path\" should not be empty") + } else { + i.To.Path = destStr + } + srcPattern, _ := variable.StringVar(vars, cp, "to", "src_pattern") + if srcPattern != "" { + pattern, err := regexp.Compile(srcPattern) + if err != nil { + return errors.Wrap(err, "\"to.pattern\" should be a valid regular expression. ") + } + i.To.Pattern = pattern + } + return nil +} + +func (i *imageCopyArgs) copy(ctx context.Context, hostVars map[string]any) error { + if sts, err := os.Stat(i.From.Path); err != nil || !sts.IsDir() { + return errors.New("\"copy.from.path\" must be a exist directory") + } + for _, img := range i.From.manifests { + img = normalizeImageNameSimple(img) + if i.To.Pattern != nil && !i.To.Pattern.MatchString(img) { + // skip + continue + } + src, err := newLocalRepository(dockerHostParser(img), i.From.Path) + if err != nil { + return err + } + dest := i.To.Path + if kkprojectv1.IsTmplSyntax(dest) { + // add temporary variable + _ = unstructured.SetNestedField(hostVars, src.Reference.Registry, "module", "image", "src", "reference", "registry") + _ = unstructured.SetNestedField(hostVars, src.Reference.Repository, "module", "image", "src", "reference", "repository") + _ = unstructured.SetNestedField(hostVars, src.Reference.Reference, "module", "image", "src", "reference", "reference") + dest, err = tmpl.ParseFunc(hostVars, dest, func(b []byte) string { return string(b) }) + if err != nil { + return err + } + } + dst, err := newLocalRepository(filepath.Join(src.Reference.Registry, src.Reference.Repository)+":"+src.Reference.Reference, dest) + if err != nil { + return err + } + if _, err = oras.Copy(ctx, src, src.Reference.Reference, dst, dst.Reference.Reference, oras.DefaultCopyOptions); err != nil { + return errors.Wrapf(err, "failed to push image %q to remote", img) + } + } + + return nil +} + // newImageArgs creates a new imageArgs instance from raw configuration func newImageArgs(_ context.Context, raw runtime.RawExtension, vars map[string]any) (*imageArgs, error) { ia := &imageArgs{} @@ -393,6 +503,22 @@ func newImageArgs(_ context.Context, raw runtime.RawExtension, vars map[string]a ia.push = ips } + if cpArgs, ok := args["copy"]; ok { + cp, ok := cpArgs.(map[string]any) + if !ok { + return nil, errors.New("\"copy\" should be map") + } + + cps := &imageCopyArgs{} + + err := cps.parseFromVars(vars, cp) + if err != nil { + return nil, err + } + + ia.copy = cps + } + return ia, nil } @@ -422,6 +548,12 @@ func ModuleImage(ctx context.Context, options ExecOptions) (string, string, erro } } + if ia.copy != nil { + if err := ia.copy.copy(ctx, ha); err != nil { + return StdoutFailed, "failed to push image", err + } + } + return StdoutSuccess, "", nil }