15308555518/ kubekey
1
0
Fork 0
mirror of https://github.com/kubesphere/kubekey.git synced 2025-12-26 01:22:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1511 from 24sama/master 

fix: add a http checksum implementation
This commit is contained in:
KubeSphere CI Bot 2022-09-29 14:15:24 +08:00 committed by GitHub
parent 68bd1ddd41 8b119b636b
commit b2eb6500b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 515 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
exp/cluster-api-provider-kubekey/api/v1beta1/component_types.go
View File

@ -51,5 +51,16 @@ type Override struct {
// Checksum is the SHA256 checksum of the binary.
// +optional
Checksum string `json:"checksum,omitempty"`
Checksum Checksum `json:"checksum,omitempty"`
}
// Checksum is the SHA256 checksum of the binary.
type Checksum struct {
// Value is the checksum string value.
// +optional
Value string `json:"value,omitempty"`
// Path defines the URL path, which is the path of the checksum file.
// +optional
Path string `json:"path,omitempty"`
}

16
exp/cluster-api-provider-kubekey/api/v1beta1/zz_generated.deepcopy.go
View File

@ -53,6 +53,21 @@ func (in *Auth) DeepCopy() *Auth {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Checksum) DeepCopyInto(out *Checksum) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Checksum.
func (in *Checksum) DeepCopy() *Checksum {
if in == nil {
return nil
}
out := new(Checksum)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Component) DeepCopyInto(out *Component) {
*out = *in
@ -706,6 +721,7 @@ func (in *Nodes) DeepCopy() *Nodes {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Override) DeepCopyInto(out *Override) {
*out = *in
out.Checksum = in.Checksum
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Override.

10
exp/cluster-api-provider-kubekey/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml
View File

@ -73,7 +73,15 @@ spec:
type: string
checksum:
description: Checksum is the SHA256 checksum of the binary.
type: string
properties:
path:
description: Path defines the URL path, which is the
path of the checksum file.
type: string
value:
description: Value is the checksum string value.
type: string
type: object
id:
description: ID is the component id name. e.g. kubeadm,
kubelet, containerd, etc.

10
exp/cluster-api-provider-kubekey/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml
View File

@ -93,7 +93,15 @@ spec:
checksum:
description: Checksum is the SHA256 checksum of
the binary.
type: string
properties:
path:
description: Path defines the URL path, which
is the path of the checksum file.
type: string
value:
description: Value is the checksum string value.
type: string
type: object
id:
description: ID is the component id name. e.g. kubeadm,
kubelet, containerd, etc.

22
exp/cluster-api-provider-kubekey/pkg/service/binary/binary.go
View File

@ -33,19 +33,19 @@ var f embed.FS
// DownloadAll downloads all binaries.
func (s *Service) DownloadAll(timeout time.Duration) error {
kubeadm, err := s.getKubeadmService(s.SSHClient, s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
kubeadm, err := s.getKubeadmService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
if err != nil {
return err
}
kubelet, err := s.getKubeletService(s.SSHClient, s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
kubelet, err := s.getKubeletService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
if err != nil {
return err
}
kubecni, err := s.getKubecniService(s.SSHClient, file.KubecniDefaultVersion, s.instanceScope.Arch())
kubecni, err := s.getKubecniService(file.KubecniDefaultVersion, s.instanceScope.Arch())
if err != nil {
return err
}
kubectl, err := s.getKubectlService(s.SSHClient, s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
kubectl, err := s.getKubectlService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
if err != nil {
return err
}
@ -69,7 +69,7 @@ func (s *Service) DownloadAll(timeout time.Duration) error {
"url", b.URL().String())
override := overrideMap[b.ID()+b.Version()+b.Arch()]
if err := util.DownloadAndCopy(b, zone, host, override.Path, override.URL, override.Checksum, timeout); err != nil {
if err := util.DownloadAndCopy(b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil {
return err
}
if err := b.Chmod("+x"); err != nil {
@ -77,7 +77,7 @@ func (s *Service) DownloadAll(timeout time.Duration) error {
}
}
if _, err := s.SSHClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(kubecni.RemotePath()), kubecni.RemotePath()); err != nil {
if _, err := s.sshClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(kubecni.RemotePath()), kubecni.RemotePath()); err != nil {
return err
}
@ -86,12 +86,12 @@ func (s *Service) DownloadAll(timeout time.Duration) error {
// ConfigureKubelet configures kubelet.
func (s *Service) ConfigureKubelet() error {
kubelet, err := s.getKubeletService(s.SSHClient, s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
kubelet, err := s.getKubeletService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
if err != nil {
return err
}
if _, err := s.SSHClient.SudoCmdf("ln -snf %s /usr/bin/kubelet", kubelet.RemotePath()); err != nil {
if _, err := s.sshClient.SudoCmdf("ln -snf %s /usr/bin/kubelet", kubelet.RemotePath()); err != nil {
return err
}
@ -137,7 +137,7 @@ func (s *Service) ConfigureKubelet() error {
return err
}
if _, err := s.SSHClient.SudoCmdf("systemctl disable kubelet && systemctl enable kubelet"); err != nil {
if _, err := s.sshClient.SudoCmdf("systemctl disable kubelet && systemctl enable kubelet"); err != nil {
return err
}
return nil
@ -145,12 +145,12 @@ func (s *Service) ConfigureKubelet() error {
// ConfigureKubeadm configures kubeadm.
func (s *Service) ConfigureKubeadm() error {
kubeadm, err := s.getKubeadmService(s.SSHClient, s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
kubeadm, err := s.getKubeadmService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch())
if err != nil {
return err
}
if _, err := s.SSHClient.SudoCmdf("ln -snf %s /usr/bin/kubeadm", kubeadm.RemotePath()); err != nil {
if _, err := s.sshClient.SudoCmdf("ln -snf %s /usr/bin/kubeadm", kubeadm.RemotePath()); err != nil {
return err
}
return nil

32
exp/cluster-api-provider-kubekey/pkg/service/binary/service.go
View File

@ -28,7 +28,7 @@ import (
// Service holds a collection of interfaces.
// The interfaces are broken down like this to group functions together.
type Service struct {
SSHClient ssh.Interface
sshClient ssh.Interface
scope scope.KKInstanceScope
instanceScope *scope.InstanceScope
@ -42,7 +42,7 @@ type Service struct {
// NewService returns a new service given the remote instance.
func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *Service {
return &Service{
SSHClient: sshClient,
sshClient: sshClient,
scope: scope,
instanceScope: instanceScope,
}
@ -50,35 +50,35 @@ func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceSc
func (s *Service) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) {
if s.templateFactory != nil {
return s.templateFactory(s.SSHClient, template, data, dst)
return s.templateFactory(s.sshClient, template, data, dst)
}
return file.NewTemplate(s.SSHClient, s.scope.RootFs(), template, data, dst)
return file.NewTemplate(s.sshClient, s.scope.RootFs(), template, data, dst)
}
func (s *Service) getKubeadmService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) {
func (s *Service) getKubeadmService(version, arch string) (operation.Binary, error) {
if s.kubeadmFactory != nil {
return s.kubeadmFactory(sshClient, version, arch)
return s.kubeadmFactory(s.sshClient, version, arch)
}
return file.NewKubeadm(sshClient, s.scope.RootFs(), version, arch)
return file.NewKubeadm(s.sshClient, s.scope.RootFs(), version, arch)
}
func (s *Service) getKubeletService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) {
func (s *Service) getKubeletService(version, arch string) (operation.Binary, error) {
if s.kubeletFactory != nil {
return s.kubeletFactory(sshClient, version, arch)
return s.kubeletFactory(s.sshClient, version, arch)
}
return file.NewKubelet(sshClient, s.scope.RootFs(), version, arch)
return file.NewKubelet(s.sshClient, s.scope.RootFs(), version, arch)
}
func (s *Service) getKubecniService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) {
func (s *Service) getKubecniService(version, arch string) (operation.Binary, error) {
if s.kubecniFactory != nil {
return s.kubecniFactory(sshClient, version, arch)
return s.kubecniFactory(s.sshClient, version, arch)
}
return file.NewKubecni(sshClient, s.scope.RootFs(), version, arch)
return file.NewKubecni(s.sshClient, s.scope.RootFs(), version, arch)
}
func (s *Service) getKubectlService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) {
func (s *Service) getKubectlService(version, arch string) (operation.Binary, error) {
if s.kubectlFactory != nil {
return s.kubectlFactory(sshClient, version, arch)
return s.kubectlFactory(s.sshClient, version, arch)
}
return file.NewKubectl(sshClient, s.scope.RootFs(), version, arch)
return file.NewKubectl(s.sshClient, s.scope.RootFs(), version, arch)
}

2
exp/cluster-api-provider-kubekey/pkg/service/containermanager/containerd.go
View File

@ -143,7 +143,7 @@ func (s *ContainerdService) Get(timeout time.Duration) error {
"url", b.URL().String())
override := overrideMap[b.ID()+b.Version()+b.Arch()]
if err := util.DownloadAndCopy(b, zone, host, override.Path, override.URL, override.Checksum, timeout); err != nil {
if err := util.DownloadAndCopy(b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil {
return err
}
}

63
exp/cluster-api-provider-kubekey/pkg/service/operation/file/binary.go
View File

@ -43,35 +43,38 @@ const (
// BinaryParams represents the parameters of a Binary.
type BinaryParams struct {
File *File
ID string
Version string
Arch string
URL *url.URL
Checksum checksum.Interface
File *File
ID string
Version string
Arch string
URL *url.URL
ChecksumList []checksum.Interface
}
// NewBinary returns a new Binary.
func NewBinary(params BinaryParams) *Binary {
b := &Binary{
file: params.File,
id: params.ID,
version: params.Version,
arch: params.Arch,
url: params.URL,
checksum: params.Checksum,
file: params.File,
id: params.ID,
version: params.Version,
arch: params.Arch,
url: params.URL,
checksumList: checksum.NewChecksums(checksum.NewInternalChecksum(params.ID, params.Version, params.Arch)),
}
if len(params.ChecksumList) != 0 {
b.checksumList.Append(params.ChecksumList...)
}
return b
}
// Binary is a binary implementation of Binary interface.
type Binary struct {
file *File
id string
version string
arch string
url *url.URL
checksum checksum.Interface
file *File
id string
version string
arch string
url *url.URL
checksumList checksum.List
}
// Name returns the name of the Binary file.
@ -111,11 +114,11 @@ func (b *Binary) RemoteExist() bool {
return false
}
if err := b.checksum.Get(); err != nil {
if err := b.checksumList.Get(); err != nil {
return false
}
if remoteSHA256 != b.checksum.Value() {
if remoteSHA256 != b.checksumList.Value() {
return false
}
return true
@ -200,12 +203,12 @@ func (b *Binary) SetZone(zone string) {
}
}
// SetChecksum sets the checksum of the binary.
func (b *Binary) SetChecksum(c checksum.Interface) {
// AppendChecksum appends a checksum to the checksum list.
func (b *Binary) AppendChecksum(c checksum.Interface) {
if reflect.ValueOf(c).IsNil() {
return
}
b.checksum = c
b.checksumList.Append(c)
}
// Get downloads the binary from remote.
@ -215,7 +218,7 @@ func (b *Binary) Get(timeout time.Duration) error {
}
if err := client.GetFile(b.LocalPath(), b.URL()); err != nil {
return errors.Wrapf(err, "failed to http get file: %s", b.LocalPath())
return errors.Wrapf(err, "failed to http get file: %s", b.URL())
}
return nil
@ -238,17 +241,13 @@ func (b *Binary) SHA256() (string, error) {
// CompareChecksum compares the checksum of the binary.
func (b *Binary) CompareChecksum() error {
if err := b.checksum.Get(); err != nil {
if err := b.checksumList.Get(); err != nil {
return errors.Wrapf(err, "%s get checksum failed", b.Name())
}
sum, err := b.SHA256()
if err != nil {
return errors.Wrapf(err, "%s caculate SHA256 failed", b.Name())
}
if sum != b.checksum.Value() {
return errors.Errorf("SHA256 no match. file: %s sha256: %s not equal checksum: %s", b.Name(), sum, b.checksum.Value())
sum := b.file.rootFs.Fs().SHA256Sum(b.LocalPath())
if sum != b.checksumList.Value() {
return errors.Errorf("SHA256 no match. file: %s sha256: %s not equal checksum: %s", b.Name(), sum, b.checksumList.Value())
}
return nil
}

60
exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum/checksums.go Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package checksum
import (
kerrors "k8s.io/apimachinery/pkg/util/errors"
)
// Checksums is a list of checksums.
type Checksums struct {
checksums []Interface
value string
}
// NewChecksums returns a new Checksums.
func NewChecksums(checksums ...Interface) *Checksums {
return &Checksums{checksums: checksums}
}
// Get gets the checksums. It will iterate through the Get() of the []Interface array and store the first fetched value.
func (c *Checksums) Get() error {
if c.value != "" {
return nil
}
var errs []error
for _, v := range c.checksums {
if err := v.Get(); err != nil {
errs = append(errs, err)
continue
}
c.value = v.Value()
return nil
}
return kerrors.NewAggregate(errs)
}
// Value returns the checksums.
func (c *Checksums) Value() string {
return c.value
}
// Append appends checksums.
func (c *Checksums) Append(checksums ...Interface) {
c.checksums = append(c.checksums, checksums...)
}

156
exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum/http_checksum.go Normal file
View File

@ -0,0 +1,156 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package checksum
import (
"bufio"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/hashicorp/go-getter"
"github.com/pkg/errors"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
)
// HTTPChecksum is a checksum that is downloaded from a URL.
type HTTPChecksum struct {
fs rootfs.Interface
url *url.URL
FileName string
value string
}
// NewHTTPChecksum returns a new HTTPChecksum.
func NewHTTPChecksum(url *url.URL, isoName string, fs rootfs.Interface) *HTTPChecksum {
return &HTTPChecksum{
url: url,
FileName: isoName,
fs: fs,
}
}
// SetHost sets the host of the URL.
func (h *HTTPChecksum) SetHost(host string) {
if host == "" {
return
}
u, err := url.Parse(host)
if err != nil {
return
}
u.Path = h.url.Path
h.url = u
}
// SetPath sets the URL path of the binary.
func (h *HTTPChecksum) SetPath(pathStr string) {
if pathStr == "" {
return
}
ref, err := url.Parse(pathStr)
if err != nil {
return
}
h.url = h.url.ResolveReference(ref)
}
// Get downloads the checksum file and parses it.
func (h *HTTPChecksum) Get() error {
tempfile, err := h.fs.Fs().MkLocalTmpFile(h.fs.ClusterRootFsDir(), filepath.Base(h.url.Path))
if err != nil {
return err
}
defer func() {
_ = h.fs.Fs().RemoveAll(tempfile)
}()
client := &getter.HttpGetter{
ReadTimeout: 15 * time.Second,
}
if err := client.GetFile(tempfile, h.url); err != nil {
return errors.Wrapf(err, "failed to http get file: %s", h.url)
}
f, err := os.Open(filepath.Clean(tempfile))
if err != nil {
return fmt.Errorf("error opening downloaded file: %s", err)
}
defer f.Close()
rd := bufio.NewReader(f)
for {
line, err := rd.ReadString('\n')
if err != nil {
if err != io.EOF {
return fmt.Errorf("error reading checksum file: %s", err)
}
if line == "" {
break
}
// parse the line, if we hit EOF, but the line is not empty
}
checksum, filename, err := parseChecksumLine(line)
if err != nil || checksum == "" {
continue
}
if filename == h.FileName {
h.value = checksum
return nil
}
}
return fmt.Errorf("no checksum found in: %s", h.url.String())
}
// Value returns the checksum value.
func (h *HTTPChecksum) Value() string {
return h.value
}
// parseChecksumLine takes a line from a checksum file and returns the checksum and filename.
func parseChecksumLine(line string) (string, string, error) {
parts := strings.Fields(line)
switch len(parts) {
case 4:
// BSD-style checksum:
// MD5 (file1) = <checksum>
// MD5 (file2) = <checksum>
if len(parts[1]) <= 2 ||
parts[1][0] != '(' || parts[1][len(parts[1])-1] != ')' {
return "", "", fmt.Errorf(
"unexpected BSD-style-checksum filename format: %s", line)
}
filename := parts[1][1 : len(parts[1])-1]
return parts[3], filename, nil
case 2:
// GNU-style:
// <checksum> file1
// <checksum> *file2
return parts[0], parts[1], nil
case 0:
return "", "", nil
default:
return parts[0], "", nil
}
}

7
exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum/interface.go
View File

@ -22,7 +22,8 @@ type Interface interface {
Value() string
}
// NewChecksum returns a new checksum implementation.
func NewChecksum(id, version, arch string) Interface {
return NewInternalChecksum(id, version, arch)
// List is the interface of a list of checksums.
type List interface {
Interface
Append(checksum ...Interface)
}

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/containerd.go
View File

@ -22,7 +22,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// Containerd info
@ -53,14 +52,12 @@ func NewContainerd(sshClient ssh.Interface, rootFs rootfs.Interface, version, ar
}
u := parseURL(DefaultDownloadHost, fmt.Sprintf(ContainerdURLPathTmpl, version, version, arch))
internal := checksum.NewChecksum(ContainerdID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: ContainerdID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: ContainerdID,
Version: version,
Arch: arch,
URL: u,
})
return &Containerd{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/crictl.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// Crictl info
@ -55,14 +54,12 @@ func NewCrictl(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch s
}
u := parseURL(DefaultDownloadHost, fmt.Sprintf(CrictlURLPathTmpl, version, version, arch))
internal := checksum.NewChecksum(CrictlID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: CrictlID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: CrictlID,
Version: version,
Arch: arch,
URL: u,
})
return &Crictl{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/docker.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/util"
)
@ -59,14 +58,12 @@ func NewDocker(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch s
}
u := parseURL(DockerURL, fmt.Sprintf(DockerURLPathTmpl, util.ArchAlias(arch), version))
internal := checksum.NewChecksum(DockerID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: DockerID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: DockerID,
Version: version,
Arch: arch,
URL: u,
})
return &Docker{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/kubeadm.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// Kubeadm info
@ -55,14 +54,12 @@ func NewKubeadm(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch
}
u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubeadmURLPathTmpl, version, arch))
internal := checksum.NewChecksum(KubeadmID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: KubeadmID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: KubeadmID,
Version: version,
Arch: arch,
URL: u,
})
return &Kubeadm{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/kubecni.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// Kubecni info
@ -57,14 +56,12 @@ func NewKubecni(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch
}
u := parseURL(DefaultDownloadHost, fmt.Sprintf(KubecniURLPathTmpl, version, arch, version))
internal := checksum.NewChecksum(KubecniID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: KubecniID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: KubecniID,
Version: version,
Arch: arch,
URL: u,
})
return &Kubecni{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/kubectl.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// Kubectl info
@ -55,14 +54,12 @@ func NewKubectl(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch
}
u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubectlURLPathTmpl, version, arch))
internal := checksum.NewChecksum(KubectlID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: KubectlID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: KubectlID,
Version: version,
Arch: arch,
URL: u,
})
return &Kubectl{binary}, nil

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/kubelet.go
View File

@ -23,7 +23,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// kubelet info
@ -55,14 +54,12 @@ func NewKubelet(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch
}
u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubeletURLPathTmpl, version, arch))
internal := checksum.NewChecksum(KubeletID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: KubeletID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: KubeletID,
Version: version,
Arch: arch,
URL: u,
})
return &Kubelet{binary}, nil

32
exp/cluster-api-provider-kubekey/pkg/service/operation/file/repositoryISO.go
View File

@ -38,6 +38,7 @@ const (
// ISO is a Binary for repository ISO file.
type ISO struct {
*Binary
HTTPChecksum *checksum.HTTPChecksum
}
// NewISO returns a new repository ISO.
@ -73,17 +74,19 @@ func NewISO(sshClient ssh.Interface, rootFs rootfs.Interface, os *osrelease.Data
}
u := parseURL(DefaultDownloadHost, urlPath)
internal := checksum.NewChecksum(os.ID, os.VersionID, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: os.ID,
Version: os.VersionID,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: os.ID,
Version: os.VersionID,
Arch: arch,
URL: u,
})
return &ISO{binary}, nil
checksumURL := parseURL(DefaultDownloadHost, fmt.Sprintf(ISOURLPathTmpl, generateCheckFileName(binary.Name())))
httpChecksum := checksum.NewHTTPChecksum(checksumURL, binary.Name(), binary.file.rootFs)
binary.AppendChecksum(httpChecksum)
return &ISO{binary, httpChecksum}, nil
}
func generateISOName(os *osrelease.Data, arch string) string {
@ -95,6 +98,19 @@ func generateISOName(os *osrelease.Data, arch string) string {
fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID + os.VersionID, "debs", arch}, "-"))
case osrelease.CentosID:
fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID + os.VersionID, "rpms", arch}, "-"))
default:
fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID, os.VersionID, arch}, "-"))
}
return fileName
}
func generateCheckFileName(isoName string) string {
return isoName[0:strings.LastIndex(isoName, "-")] + ".iso.sha256sum.txt"
}
// SetZone override Binary's SetZone method.
func (i *ISO) SetZone(zone string) {
if strings.EqualFold(zone, ZONE) {
return
}
}

44
exp/cluster-api-provider-kubekey/pkg/service/operation/file/repositoryISO_test.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package file
import (
"testing"
)
func Test_generateCheckFileName(t *testing.T) {
tests := []struct {
isoName string
want string
}{
{
isoName: "centos7-rpms-amd64.iso",
want: "centos7-rpms.iso.sha256sum.txt",
},
{
isoName: "ubuntu-22.04-debs-arm64.iso",
want: "ubuntu-22.04-debs.iso.sha256sum.txt",
},
}
for _, tt := range tests {
t.Run("", func(t *testing.T) {
if got := generateCheckFileName(tt.isoName); got != tt.want {
t.Errorf("generateCheckFileName() = %v, want %v", got, tt.want)
}
})
}
}

13
exp/cluster-api-provider-kubekey/pkg/service/operation/file/runc.go
View File

@ -22,7 +22,6 @@ import (
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/rootfs"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file/checksum"
)
// runc info
@ -54,14 +53,12 @@ func NewRunc(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch str
}
u := parseURL(DefaultDownloadHost, fmt.Sprintf(RuncURLPathTmpl, version, arch))
internal := checksum.NewChecksum(RuncID, version, arch)
binary := NewBinary(BinaryParams{
File: file,
ID: RuncID,
Version: version,
Arch: arch,
URL: u,
Checksum: internal,
File: file,
ID: RuncID,
Version: version,
Arch: arch,
URL: u,
})
return &Runc{binary}, nil

2
exp/cluster-api-provider-kubekey/pkg/service/operation/interface.go
View File

@ -49,7 +49,7 @@ type Binary interface {
SetHost(host string)
SetPath(path string)
SetZone(zone string)
SetChecksum(checksum checksum.Interface)
AppendChecksum(c checksum.Interface)
Get(timeout time.Duration) error
CompareChecksum() error
}

2
exp/cluster-api-provider-kubekey/pkg/service/operation/repository/repository_deb.go
View File

@ -38,7 +38,7 @@ func NewDeb(sshClient ssh.Interface) *Debian {
// Add adds a local repository using the iso file.
func (d *Debian) Add(path string) error {
if _, err := d.SSHClient.SudoCmd(
fmt.Sprintf("echo 'deb [trusted=yes] file://%s /' "+
fmt.Sprintf("echo 'deb [trusted=yes] file://%s ./' "+
"| sudo tee /etc/apt/sources.list.d/kubekey.list > /dev/null", path)); err != nil {
return err
}

19
exp/cluster-api-provider-kubekey/pkg/service/repository/repository.go
View File

@ -18,6 +18,7 @@ package repository
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
@ -25,8 +26,10 @@ import (
"github.com/pkg/errors"
infrav1 "github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/api/v1beta1"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/repository"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/util"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/util/filesystem"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/util/osrelease"
)
@ -56,7 +59,8 @@ func (s *Service) Get(timeout time.Duration) error {
return nil
}
iso, err := s.getISOService(s.sshClient, s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO)
s.scope.V(4).Info("os release", "os", s.os)
iso, err := s.getISOService(s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO)
if err != nil {
return err
}
@ -72,7 +76,9 @@ func (s *Service) Get(timeout time.Duration) error {
"url", iso.URL().String())
override := overrideMap[iso.ID()+iso.Version()+iso.Arch()]
if err := util.DownloadAndCopy(iso, zone, host, override.Path, override.URL, override.Checksum, timeout); err != nil {
iso.HTTPChecksum.SetHost(host)
iso.HTTPChecksum.SetPath(override.Checksum.Path)
if err := util.DownloadAndCopy(iso, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil {
return err
}
return nil
@ -84,12 +90,17 @@ func (s *Service) MountISO() error {
return nil
}
iso, err := s.getISOService(s.sshClient, s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO)
mountPath := filepath.Join(file.MntDir, "repository")
dirSvc := s.getDirectoryService(mountPath, os.FileMode(filesystem.FileMode0755))
if err := dirSvc.Make(); err != nil {
return errors.Wrapf(err, "failed to make directory %s", mountPath)
}
iso, err := s.getISOService(s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO)
if err != nil {
return err
}
mountPath := filepath.Join(iso.RemotePath(), "repository")
if _, err := s.sshClient.SudoCmd(fmt.Sprintf("sudo mount -t iso9660 -o loop %s %s", iso.RemotePath(), mountPath)); err != nil {
return errors.Wrapf(err, "mount %s at %s failed", iso.RemotePath(), mountPath)
}

19
exp/cluster-api-provider-kubekey/pkg/service/repository/service.go
View File

@ -17,9 +17,12 @@
package repository
import (
"os"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/clients/ssh"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/scope"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/directory"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/file"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/service/operation/repository"
"github.com/kubesphere/kubekey/exp/cluster-api-provider-kubekey/pkg/util/osrelease"
@ -36,7 +39,8 @@ type Service struct {
mountPath string
repositoryFactory func(sshClient ssh.Interface, os *osrelease.Data) operation.Repository
isoFactory func(sshClient ssh.Interface, arch, isoName string) (operation.Binary, error)
isoFactory func(sshClient ssh.Interface, arch, isoName string) (*file.ISO, error)
directoryFactory func(sshClient ssh.Interface, path string, mode os.FileMode) operation.Directory
}
// NewService returns a new service given the remote instance kubekey build-in repository client.
@ -55,9 +59,16 @@ func (s *Service) getRepositoryService(os *osrelease.Data) operation.Repository
return repository.NewService(s.sshClient, os)
}
func (s *Service) getISOService(sshClient ssh.Interface, os *osrelease.Data, arch string, isoName string) (operation.Binary, error) {
func (s *Service) getISOService(os *osrelease.Data, arch string, isoName string) (*file.ISO, error) {
if s.isoFactory != nil {
return s.isoFactory(sshClient, arch, isoName)
return s.isoFactory(s.sshClient, arch, isoName)
}
return file.NewISO(sshClient, s.scope.RootFs(), os, arch, isoName)
return file.NewISO(s.sshClient, s.scope.RootFs(), os, arch, isoName)
}
func (s *Service) getDirectoryService(path string, mode os.FileMode) operation.Directory {
if s.directoryFactory != nil {
return s.directoryFactory(s.sshClient, path, mode)
}
return directory.NewService(s.sshClient, path, mode)
}

2
exp/cluster-api-provider-kubekey/pkg/service/util/downloader.go
View File

@ -29,7 +29,7 @@ func DownloadAndCopy(b operation.Binary, zone, host, path, url, checksumStr stri
return nil
}
b.SetChecksum(checksum.NewStringChecksum(checksumStr))
b.AppendChecksum(checksum.NewStringChecksum(checksumStr))
if !(b.LocalExist() && b.CompareChecksum() == nil) {
// Only the host is an empty string, we can set up the zone.
// Because the URL path which in the QingStor is not the same as the default.

21
exp/cluster-api-provider-kubekey/pkg/util/filesystem/fs.go
View File

@ -43,7 +43,7 @@ func (f FileSystem) MkdirAll(path string) error {
return os.MkdirAll(path, os.ModePerm)
}
// MD5Sum returns the file MD5 sum for the given local path
// MD5Sum returns the file MD5 sum for the given local path.
func (f FileSystem) MD5Sum(localPath string) string {
md5, err := hash.FileMD5(localPath)
if err != nil {
@ -52,6 +52,15 @@ func (f FileSystem) MD5Sum(localPath string) string {
return md5
}
// SHA256Sum returns the file SHA256 sum for the given local path.
func (f FileSystem) SHA256Sum(localPath string) string {
sha256, err := hash.FileSHA256(localPath)
if err != nil {
return ""
}
return sha256
}
// MkLocalTmpDir creates a temporary directory and returns the path
func (f FileSystem) MkLocalTmpDir() (string, error) {
tempDir, err := ioutil.TempDir(DefaultLocalTmpDir, ".Tmp-")
@ -61,6 +70,16 @@ func (f FileSystem) MkLocalTmpDir() (string, error) {
return tempDir, os.MkdirAll(tempDir, os.ModePerm)
}
// MkLocalTmpFile creates a temporary file and returns the path.
func (f FileSystem) MkLocalTmpFile(dir, pattern string) (string, error) {
file, err := ioutil.TempFile(dir, pattern)
if err != nil {
return "", err
}
_ = file.Close()
return file.Name(), nil
}
// RemoveAll the same as os.RemoveAll().
func (f FileSystem) RemoveAll(path ...string) error {
for _, fi := range path {

4
exp/cluster-api-provider-kubekey/pkg/util/filesystem/interface.go
View File

@ -28,8 +28,12 @@ type Interface interface {
MkdirAll(path string) error
// MD5Sum returns the file MD5 sum for the given local path.
MD5Sum(localPath string) string
// SHA256Sum returns the file SHA256 sum for the given local path.
SHA256Sum(localPath string) string
// MkLocalTmpDir creates a temporary directory and returns the path.
MkLocalTmpDir() (string, error)
// MkLocalTmpFile creates a temporary file and returns the path.
MkLocalTmpFile(dir, pattern string) (string, error)
// RemoveAll the same as os.RemoveAll().
RemoveAll(path ...string) error
}

18
exp/cluster-api-provider-kubekey/pkg/util/hash/hash.go
View File

@ -19,6 +19,7 @@ package hash
import (
"crypto/md5" // #nosec
"crypto/sha256"
"fmt"
"io"
"os"
@ -31,6 +32,7 @@ func FileMD5(path string) (string, error) {
if err != nil {
return "", err
}
defer file.Close()
m := md5.New() // #nosec
if _, err := io.Copy(m, file); err != nil {
@ -40,3 +42,19 @@ func FileMD5(path string) (string, error) {
fileMd5 := fmt.Sprintf("%x", m.Sum(nil))
return fileMd5, nil
}
// FileSHA256 count file sha256
func FileSHA256(path string) (string, error) {
file, err := os.Open(filepath.Clean(path))
if err != nil {
return "", err
}
defer file.Close()
s := sha256.New()
if _, err := io.Copy(s, file); err != nil {
return "", err
}
return fmt.Sprintf("%x", s.Sum(nil)), nil
}

3
exp/cluster-api-provider-kubekey/pkg/util/osrelease/osrelease.go
View File

@ -126,6 +126,9 @@ func parseLine(line string) (string, string, error) {
value = strings.Replace(value, `\$`, `$`, -1)
value = strings.Replace(value, `\\`, `\`, -1)
value = strings.Replace(value, "\\`", "`", -1)
value = strings.TrimRight(value, "\r\n")
value = strings.TrimLeft(value, "\"")
value = strings.TrimRight(value, "\"")
return key, value, nil
}

3
exp/cluster-api-provider-kubekey/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml
View File

@ -50,7 +50,8 @@ spec:
roles:
- control-plane
repository:
iso: "none"
iso: "auto"
update: true
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.

3
exp/cluster-api-provider-kubekey/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml
View File

@ -10,7 +10,8 @@ spec:
- control-plane
- worker
repository:
iso: "none"
iso: "auto"
update: true
---
# KubeadmConfigTemplate referenced by the MachineDeployment
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1