From d9c699f80a7992a924ce66933a63d0c34ccf35d1 Mon Sep 17 00:00:00 2001 From: zuoxuesong-worker Date: Mon, 17 Nov 2025 16:29:34 +0800 Subject: [PATCH] feat: feat no root ssh (#2858) feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh feat: feat no root ssh Signed-off-by: xuesongzuo@yunify.com --- .../install/cloud-config/tasks/main.yaml | 9 +- .../kubernetes/tasks/kubernetes.yaml | 3 +- .../certs/renew/kubernetes/tasks/kube.yaml | 10 +- .../tasks/init_kubernetes.yaml | 4 + .../join-kubernetes/tasks/main.yaml | 8 +- .../kubernetes/tasks/kubernetes.yaml | 3 +- pkg/connector/ssh_connector.go | 125 ++++++++++++------ pkg/modules/copy.go | 40 +++++- pkg/modules/fetch.go | 16 ++- pkg/modules/template.go | 29 +++- 10 files changed, 187 insertions(+), 60 deletions(-) diff --git a/builtin/capkk/roles/install/cloud-config/tasks/main.yaml b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml index e5394b27..0b75fc66 100644 --- a/builtin/capkk/roles/install/cloud-config/tasks/main.yaml +++ b/builtin/capkk/roles/install/cloud-config/tasks/main.yaml @@ -130,7 +130,14 @@ loop: "{{ .cloud_config.runcmd | toJson }}" command: "{{ .item }}" -- name: Sync kubeconfig +- name: Sync kubeconfig to current user + copy: + src: >- + {{ .cloud_config_dir }}/kubeconfig/value + dest: ~/.kube/config + mode: 0600 + +- name: Sync kubeconfig to root copy: src: >- {{ .cloud_config_dir }}/kubeconfig/value diff --git a/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml b/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml index e68ae092..3031a616 100644 --- a/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml +++ b/builtin/capkk/roles/uninstall/kubernetes/tasks/kubernetes.yaml @@ -29,5 +29,6 @@ rm -rf /usr/local/bin/kubeadm && rm -rf /usr/local/bin/kubelet && rm -rf /usr/local/bin/kubectl rm -rf /var/lib/kubelet/ rm -rf /etc/kubernetes/ - rm -rf .kube/config + rm -rf ~/.kube/config + rm -rf /root/.kube/config rm -rf /var/lib/etcd \ No newline at end of file diff --git a/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml b/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml index 5e3ca321..5c219c14 100644 --- a/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml +++ b/builtin/core/roles/certs/renew/kubernetes/tasks/kube.yaml @@ -42,8 +42,14 @@ dest: >- {{ .binary_dir }}/kubeconfig -- name: Kubernetes | Distribute kubeconfig to remote host +- name: Kubernetes | Distribute kubeconfig to remote host of current user copy: src: >- {{ .binary_dir }}/kubeconfig - dest: /root/.kube/config + dest: ~/.kube/config + +- name: Kubernetes | Distribute kubeconfig to remote host of root user + copy: + src: >- + {{ .binary_dir }}/kubeconfig + dest: /root/.kube/config \ No newline at end of file diff --git a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml index 8c2e5136..4e445083 100644 --- a/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml +++ b/builtin/core/roles/kubernetes/init-kubernetes/tasks/init_kubernetes.yaml @@ -45,9 +45,13 @@ - name: Init | Copy kubeconfig to default directory command: | + if [ ! -d ~/.kube ]; then + mkdir -p ~/.kube + fi if [ ! -d /root/.kube ]; then mkdir -p /root/.kube fi + cp -f /etc/kubernetes/admin.conf ~/.kube/config cp -f /etc/kubernetes/admin.conf /root/.kube/config when: .kubernetes_install_LoadState.stdout | eq "not-found" diff --git a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml index a3a7fcb8..682c1863 100644 --- a/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml +++ b/builtin/core/roles/kubernetes/join-kubernetes/tasks/main.yaml @@ -13,7 +13,13 @@ command: | /usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull -- name: Join | Synchronize kubeconfig to remote node +- name: Join | Synchronize kubeconfig to remote node for current user + copy: + src: >- + {{ .work_dir }}/kubekey/kubeconfig + dest: ~/.kube/config + +- name: Join | Synchronize kubeconfig to remote node for root copy: src: >- {{ .work_dir }}/kubekey/kubeconfig diff --git a/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml b/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml index 083555fb..b21504fa 100644 --- a/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml +++ b/builtin/core/roles/uninstall/kubernetes/tasks/kubernetes.yaml @@ -22,5 +22,6 @@ # If /var/log/pods/ is not cleaned up, static pods may accumulate unexpected restarts due to lingering log files interfering with their lifecycle. rm -rf /var/log/pods/ rm -rf /etc/kubernetes/ - rm -rf .kube/config + rm -rf ~/.kube/config + rm -rf /root/.kube/config rm -rf /var/lib/etcd \ No newline at end of file diff --git a/pkg/connector/ssh_connector.go b/pkg/connector/ssh_connector.go index 61a37087..349154df 100644 --- a/pkg/connector/ssh_connector.go +++ b/pkg/connector/ssh_connector.go @@ -17,6 +17,7 @@ limitations under the License. package connector import ( + "bufio" "bytes" "context" "fmt" @@ -26,6 +27,7 @@ import ( "os/user" "path/filepath" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -120,6 +122,8 @@ type sshConnector struct { shell string gatherFacts *cacheGatherFact + + mu sync.Mutex } // Init connector, get ssh.Client @@ -188,6 +192,33 @@ func (c *sshConnector) Close(context.Context) error { return c.client.Close() } +func (c *sshConnector) session() (*ssh.Session, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.client == nil { + return nil, errors.New("connection closed") + } + + sess, err := c.client.NewSession() + if err != nil { + return nil, err + } + + modes := ssh.TerminalModes{ + ssh.ECHO: 0, // disable echoing + ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud + ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud + } + + err = sess.RequestPty("xterm", 100, 50, modes) + if err != nil { + return nil, err + } + + return sess, nil +} + // PutFile to remote node. src is the file bytes. dst is the remote filename func (c *sshConnector) PutFile(_ context.Context, src []byte, dst string, mode fs.FileMode) error { // create sftp client @@ -243,72 +274,78 @@ func (c *sshConnector) FetchFile(_ context.Context, src string, dst io.Writer) e return nil } -// ExecuteCommand in remote host +// ExecuteCommand exec cmd with sudo func (c *sshConnector) ExecuteCommand(_ context.Context, cmd string) ([]byte, []byte, error) { - cmd = fmt.Sprintf("sudo -SE %s << 'KUBEKEY_EOF'\n%s\nKUBEKEY_EOF\n", c.shell, cmd) - klog.V(5).InfoS("exec ssh command", "cmd", cmd, "host", c.Host) - // create ssh session - session, err := c.client.NewSession() + session, err := c.session() if err != nil { - return nil, nil, errors.Wrap(err, "failed to create ssh session") + return nil, nil, err } defer session.Close() - // get pipe from session - stdin, err := session.StdinPipe() + cmd = SudoPrefix(c.shell, cmd) + + in, err := session.StdinPipe() if err != nil { return nil, nil, errors.Wrap(err, "failed to get stdin pipe") } - stdout, err := session.StdoutPipe() + + out, err := session.StdoutPipe() if err != nil { return nil, nil, errors.Wrap(err, "failed to get stdout pipe") } + stderr, err := session.StderrPipe() if err != nil { return nil, nil, errors.Wrap(err, "failed to get stderr pipe") } - // Start the remote command - if err := session.Start(cmd); err != nil { + + if err = session.Start(cmd); err != nil { return nil, nil, errors.Wrap(err, "failed to start session") } - if c.Password != "" { - if _, err := stdin.Write([]byte(c.Password + "\n")); err != nil { - return nil, nil, errors.Wrap(err, "failed to write password") + var ( + output []byte + line = "" + r = bufio.NewReader(out) + ) + + for { + b, err := r.ReadByte() + if err != nil { + break + } + + output = append(output, b) + + if b == byte('\n') { + line = "" + continue + } + + line += string(b) + + if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { + _, err = in.Write([]byte(c.Password + "\n")) + if err != nil { + break + } } } - if err := stdin.Close(); err != nil { - return nil, nil, errors.Wrap(err, "failed to close stdin pipe") - } - // Create buffers to store stdout and stderr output - var stdoutBuf, stderrBuf bytes.Buffer - - // When reading large amounts of data from stdout/stderr, the pipe buffer can fill up - // and block the remote command from completing if we don't read from it continuously. - // To prevent this deadlock scenario, we need to read stdout/stderr asynchronously - // in separate goroutines while the command is running. - // Create channels to signal when copying is complete - stdoutDone := make(chan error, 1) - stderrDone := make(chan error, 1) - - // Copy stdout and stderr concurrently to prevent pipe buffer from filling - go func() { - _, err := io.Copy(&stdoutBuf, stdout) - stdoutDone <- err - }() - go func() { - _, err := io.Copy(&stderrBuf, stderr) - stderrDone <- err - }() - - // Wait for command to complete + outStr := strings.TrimPrefix(string(output), fmt.Sprintf("[sudo] password for %s:", c.User)) err = session.Wait() + var stderrBuffer bytes.Buffer + _, _ = io.Copy(&stderrBuffer, stderr) + outStr = strings.TrimSpace(outStr) + stderrData := stderrBuffer.Bytes() + if err != nil { + return []byte(outStr), nil, errors.Wrap(err, strings.TrimSpace(string(stderrData))) + } + return []byte(outStr), stderrData, nil +} - // Wait for stdout and stderr copying to finish to ensure we've captured all output - <-stdoutDone - <-stderrDone - - return stdoutBuf.Bytes(), stderrBuf.Bytes(), errors.Wrap(err, "failed to execute ssh command") +// SudoPrefix returns the prefix for sudo commands. +func SudoPrefix(shell, cmd string) string { + return fmt.Sprintf("TERM=dumb; export LANG=C.UTF-8;sudo -E %s << 'KUBEKEY_EOF'\n%s\nKUBEKEY_EOF", shell, cmd) } // HostInfo from gatherFacts cache diff --git a/pkg/modules/copy.go b/pkg/modules/copy.go index ef4c3075..9f8c3ed4 100644 --- a/pkg/modules/copy.go +++ b/pkg/modules/copy.go @@ -18,6 +18,7 @@ package modules import ( "context" + "fmt" "io/fs" "math" "os" @@ -251,7 +252,15 @@ func (ca copyArgs) copyAbsoluteDir(ctx context.Context, conn connector.Connector dest = filepath.Join(ca.dest, rel) } - return conn.PutFile(ctx, data, dest, mode) + tmpDest := filepath.Join("/tmp", ca.dest) + + if err = conn.PutFile(ctx, data, tmpDest, mode); err != nil { + return err + } + + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err }) } @@ -289,8 +298,16 @@ func (ca copyArgs) copyRelativeDir(ctx context.Context, pj project.Project, relP } dest = filepath.Join(ca.dest, rel) } + tmpDest := filepath.Join("/tmp", ca.dest) - return conn.PutFile(ctx, data, dest, mode) + err = conn.PutFile(ctx, data, tmpDest, mode) + if err != nil { + return err + } + + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err }) } @@ -305,7 +322,15 @@ func (ca copyArgs) copyContent(ctx context.Context, mode fs.FileMode, conn conne mode = os.FileMode(*ca.mode) } - if err := conn.PutFile(ctx, []byte(ca.content), ca.dest, mode); err != nil { + tmpDest := filepath.Join("/tmp", ca.dest) + + if err := conn.PutFile(ctx, []byte(ca.content), tmpDest, mode); err != nil { + return StdoutFailed, "failed to copy file", err + } + + _, _, err := conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(ca.dest), tmpDest, ca.dest)) + + if err != nil { return StdoutFailed, "failed to copy file", err } @@ -323,8 +348,15 @@ func (ca copyArgs) copyFile(ctx context.Context, data []byte, mode fs.FileMode, if ca.mode != nil { mode = os.FileMode(*ca.mode) } + tmpDest := filepath.Join("/tmp", dest) - return conn.PutFile(ctx, data, dest, mode) + if err := conn.PutFile(ctx, data, tmpDest, mode); err != nil { + return err + } + + _, _, err := conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err } // Register the "copy" module at init. diff --git a/pkg/modules/fetch.go b/pkg/modules/fetch.go index 1ea5cfdc..be860ea4 100644 --- a/pkg/modules/fetch.go +++ b/pkg/modules/fetch.go @@ -18,9 +18,11 @@ package modules import ( "context" + "fmt" "os" "path/filepath" + "k8s.io/apimachinery/pkg/util/rand" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "github.com/kubesphere/kubekey/v4/pkg/variable" @@ -78,6 +80,10 @@ func ModuleFetch(ctx context.Context, options ExecOptions) (string, string, erro if err != nil { return StdoutFailed, "\"dest\" in args should be string", err } + tmpDir, err := variable.StringVar(ha, ha, "tmp_dir") + if err != nil || tmpDir == "" { + tmpDir = "/tmp/kubekey/" + } // get connector conn, err := options.getConnector(ctx) @@ -99,7 +105,15 @@ func ModuleFetch(ctx context.Context, options ExecOptions) (string, string, erro } defer destFile.Close() - if err := conn.FetchFile(ctx, srcParam, destFile); err != nil { + tmpFetchFileName := filepath.Join(tmpDir, fmt.Sprintf("fetch-%s-%s", options.Task.GetUID(), rand.String(5))) + + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("cp %s %s\nchmod 755 %s", srcParam, tmpFetchFileName, tmpFetchFileName)) + + if err != nil { + return StdoutFailed, "failed to fetch file", err + } + + if err = conn.FetchFile(ctx, tmpFetchFileName, destFile); err != nil { return StdoutFailed, "failed to fetch file", err } diff --git a/pkg/modules/template.go b/pkg/modules/template.go index 0c472e3e..0606fc7e 100644 --- a/pkg/modules/template.go +++ b/pkg/modules/template.go @@ -18,6 +18,7 @@ package modules import ( "context" + "fmt" "io/fs" "math" "os" @@ -226,8 +227,15 @@ func handleRelativeDir(ctx context.Context, pj project.Project, relPath string, } dest = filepath.Join(ta.dest, rel) } + tmpDest := filepath.Join("/tmp", dest) - return conn.PutFile(ctx, result, dest, mode) + if err = conn.PutFile(ctx, result, tmpDest, mode); err != nil { + return err + } + + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err }) } @@ -246,8 +254,15 @@ func (ta templateArgs) readFile(ctx context.Context, data string, mode fs.FileMo if ta.mode != nil { mode = os.FileMode(*ta.mode) } + tmpDest := filepath.Join("/tmp", dest) - return conn.PutFile(ctx, result, dest, mode) + if err = conn.PutFile(ctx, result, tmpDest, mode); err != nil { + return err + } + + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err } // absDir when template.src is absolute dir, get all files by os, parse it, and copy to remote. @@ -288,11 +303,15 @@ func (ta templateArgs) absDir(ctx context.Context, conn connector.Connector, var dest = filepath.Join(ta.dest, rel) } - if err := conn.PutFile(ctx, result, dest, mode); err != nil { - return errors.Wrap(err, "failed to put file") + tmpDest := filepath.Join("/tmp", dest) + + if err = conn.PutFile(ctx, result, tmpDest, mode); err != nil { + return err } - return nil + _, _, err = conn.ExecuteCommand(ctx, fmt.Sprintf("mkdir -p %s\nmv %s %s", filepath.Dir(dest), tmpDest, dest)) + + return err }); err != nil { return errors.Wrapf(err, "failed to walk dir %q", ta.src) }