feat: use unstructured to get or set value for config (#2519)

Signed-off-by: joyceliu <joyceliu@yunify.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

api/core/v1/config_types.go

@@ -17,13 +17,12 @@ limitations under the License.
 package v1
 
 import (
-	"reflect"
-	"strings"
+	"encoding/json"
 
 	"github.com/cockroachdb/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/json"
 )
 
 // Config store global vars for playbook.
@@ -33,70 +32,44 @@ type Config struct {
 	Spec runtime.RawExtension `json:"spec,omitempty"`
 }
 
-// SetValue to config
-// if key contains "." (a.b), will convert map and set value (a:b:value)
-func (c *Config) SetValue(key string, value any) error {
-	configMap := make(map[string]any)
-	if c.Spec.Raw != nil {
-		if err := json.Unmarshal(c.Spec.Raw, &configMap); err != nil {
-			return errors.WithStack(err)
-		}
+// UnmarshalJSON decodes spec.Raw into spec.Object
+func (c *Config) UnmarshalJSON(data []byte) error {
+	type Alias Config
+	aux := &Alias{}
+	if err := json.Unmarshal(data, aux); err != nil {
+		return errors.Wrap(err, "failed to unmarshal config")
 	}
-	// set value
-	var f func(input map[string]any, key []string, value any) any
-	f = func(input map[string]any, key []string, value any) any {
-		if len(key) == 0 {
-			return input
+	*c = Config(*aux)
+
+	// Decode spec.Raw into spec.Object if it's not already set
+	if len(c.Spec.Raw) > 0 && c.Spec.Object == nil {
+		var objMap map[string]interface{}
+		if err := json.Unmarshal(c.Spec.Raw, &objMap); err != nil {
+			return errors.Wrap(err, "failed to unmarshal spec.Raw")
 		}
-
-		firstKey := key[0]
-		if len(key) == 1 {
-			input[firstKey] = value
-
-			return input
-		}
-
-		// Handle nested maps
-		if v, ok := input[firstKey]; ok && reflect.TypeOf(v).Kind() == reflect.Map {
-			if vd, ok := v.(map[string]any); ok {
-				input[firstKey] = f(vd, key[1:], value)
-			}
-		} else {
-			input[firstKey] = f(make(map[string]any), key[1:], value)
-		}
-
-		return input
+		c.Spec.Object = &unstructured.Unstructured{Object: objMap}
 	}
-	data, err := json.Marshal(f(configMap, strings.Split(key, "."), value))
-	if err != nil {
-		return errors.Wrapf(err, "failed to marshal %q value to json", key)
-	}
-	c.Spec.Raw = data
 
 	return nil
 }
 
-// GetValue by key
-// if key contains "." (a.b), find by the key path (if a:b:value in config.and get value)
-func (c *Config) GetValue(key string) (any, error) {
-	configMap := make(map[string]any)
-	if err := json.Unmarshal(c.Spec.Raw, &configMap); err != nil {
-		return nil, errors.WithStack(err)
-	}
-	// get all value
-	if key == "" {
-		return configMap, nil
-	}
-	// get value
-	var result any = configMap
-	for _, k := range strings.Split(key, ".") {
-		r, ok := result.(map[string]any)
-		if !ok {
-			// cannot find value
-			return nil, errors.Errorf("cannot find key: %s", key)
+// MarshalJSON ensures spec.Object is converted back to spec.Raw
+func (c *Config) MarshalJSON() ([]byte, error) {
+	// Ensure spec.Object is serialized into spec.Raw
+	if c.Spec.Object != nil {
+		raw, err := json.Marshal(c.Spec.Object)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to marshal spec.Object")
 		}
-		result = r[k]
+		c.Spec.Raw = raw
 	}
 
-	return result, nil
+	type Alias Config
+	return json.Marshal((*Alias)(c))
+}
+
+// Value returns the underlying map[string]any from the Config's unstructured Object.
+// This provides direct access to the config values stored in Spec.Object.
+func (c *Config) Value() map[string]any {
+	return c.Spec.Object.(*unstructured.Unstructured).Object
 }

api/core/v1/config_types_test.go

@@ -1,112 +0,0 @@
-/*
-Copyright 2024 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"k8s.io/apimachinery/pkg/runtime"
-)
-
-func TestSetValue(t *testing.T) {
-	testcases := []struct {
-		name   string
-		key    string
-		val    any
-		except Config
-	}{
-		{
-			name:   "one level",
-			key:    "a",
-			val:    2,
-			except: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":2}`)}},
-		},
-		{
-			name:   "two level repeat key",
-			key:    "a.b",
-			val:    2,
-			except: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":{"b":2}}`)}},
-		},
-		{
-			name:   "two level no-repeat key",
-			key:    "b.c",
-			val:    2,
-			except: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1,"b":{"c":2}}`)}},
-		},
-	}
-
-	for _, tc := range testcases {
-		in := Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1}`)}}
-		t.Run(tc.name, func(t *testing.T) {
-			err := in.SetValue(tc.key, tc.val)
-			if err != nil {
-				t.Fatal(err)
-			}
-			assert.Equal(t, tc.except, in)
-		})
-	}
-}
-
-func TestGetValue(t *testing.T) {
-	testcases := []struct {
-		name   string
-		key    string
-		config Config
-		except any
-	}{
-		{
-			name:   "all value",
-			key:    "",
-			config: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1}`)}},
-			except: map[string]any{
-				"a": int64(1),
-			},
-		},
-		{
-			name:   "none value",
-			key:    "b",
-			config: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1}`)}},
-			except: nil,
-		},
-		{
-			name:   "none multi value",
-			key:    "b.c",
-			config: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1}`)}},
-			except: nil,
-		},
-		{
-			name:   "find one value",
-			key:    "a",
-			config: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":1}`)}},
-			except: int64(1),
-		},
-		{
-			name:   "find mulit value",
-			key:    "a.b",
-			config: Config{Spec: runtime.RawExtension{Raw: []byte(`{"a":{"b":1}}`)}},
-			except: int64(1),
-		},
-	}
-
-	for _, tc := range testcases {
-		t.Run(tc.name, func(t *testing.T) {
-			value, _ := tc.config.GetValue(tc.key)
-			assert.Equal(t, tc.except, value)
-		})
-	}
-}

builtin/capkk/roles/install/cloud-config/tasks/main.yaml

@@ -140,4 +140,11 @@
   loop: "{{ .kubernetes.roles | toJson }}"
   command: |
     #!/bin/bash
-    kubectl label node "{{ .hostname }}" node-role.kubernetes.io/{{ .item }}="" --overwrite
+    kubectl label node "{{ .hostname }}" node-role.kubernetes.io/{{ .item }}="" --overwrite
+
+- name: Remove traint if node is worker
+  when: .kubernetes.roles | has "worker"
+  ignore_errors: true
+  command: |
+    kubectl taint nodes "{{ .hostname }}" node-role.kubernetes.io/master-
+    kubectl taint nodes "{{ .hostname }}" node-role.kubernetes.io/control-plane-

cmd/kk/app/options/builtin/create.go

@@ -26,6 +26,7 @@ import (
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	cliflag "k8s.io/component-base/cli/flag"
 
 	"github.com/kubesphere/kubekey/v4/cmd/kk/app/options"
@@ -104,13 +105,13 @@ func (o *CreateClusterOptions) Complete(cmd *cobra.Command, args []string) (*kkc
 func (o *CreateClusterOptions) completeConfig() error {
 	if o.ContainerManager != "" {
 		// override container_manager in config
-		if err := o.CommonOptions.Config.SetValue("cri.container_manager", o.ContainerManager); err != nil {
-			return errors.WithStack(err)
+		if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.ContainerManager, "cri", "container_manager"); err != nil {
+			return errors.Wrapf(err, "failed to set %q to config", "cri.container_manager")
 		}
 	}
 
-	if err := o.CommonOptions.Config.SetValue("kube_version", o.Kubernetes); err != nil {
-		return errors.WithStack(err)
+	if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), o.Kubernetes, "kube_version"); err != nil {
+		return errors.Wrapf(err, "failed to set %q to config", "kube_version")
 	}
 
 	return nil

cmd/kk/app/options/builtin/init.go

@@ -26,6 +26,7 @@ import (
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	cliflag "k8s.io/component-base/cli/flag"
 
 	"github.com/kubesphere/kubekey/v4/cmd/kk/app/options"
@@ -85,19 +86,16 @@ func (o *InitOSOptions) Complete(cmd *cobra.Command, args []string) (*kkcorev1.P
 }
 
 func (o *InitOSOptions) complateConfig() error {
-	if workdir, err := o.CommonOptions.Config.GetValue(_const.Workdir); err != nil {
+
+	if wd, _, err := unstructured.NestedString(o.CommonOptions.Config.Value(), _const.Workdir); err != nil {
 		// workdir should set by CommonOptions
-		return errors.Wrapf(err, "failed to get value %q in config", _const.Workdir)
+		return errors.Wrapf(err, "failed to get %q in config", _const.Workdir)
 	} else {
-		wd, ok := workdir.(string)
-		if !ok {
-			return errors.New("workdir should be string value")
-		}
 		// set binary dir if not set
-		if _, err := o.CommonOptions.Config.GetValue(_const.BinaryDir); err != nil {
-			// not found set default
-			if err := o.CommonOptions.Config.SetValue(_const.BinaryDir, filepath.Join(wd, "kubekey")); err != nil {
-				return errors.Wrapf(err, "failed to set value %q in config", _const.Workdir)
+		if _, _, err := unstructured.NestedString(o.CommonOptions.Config.Value(), _const.BinaryDir); err != nil {
+			// workdir should set by CommonOptions
+			if err := unstructured.SetNestedField(o.CommonOptions.Config.Value(), filepath.Join(wd, "kubekey"), _const.BinaryDir); err != nil {
+				return errors.Wrapf(err, "failed to set %q in config", _const.Workdir)
 			}
 		}
 	}

cmd/kk/app/options/option.go

@@ -27,6 +27,7 @@ import (
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/client-go/rest"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
@@ -196,14 +197,14 @@ func (o *CommonOptions) Complete(playbook *kkcorev1.Playbook) error {
 func (o *CommonOptions) completeConfig(config *kkcorev1.Config) error {
 	// set value by command args
 	if o.Workdir != "" {
-		if err := config.SetValue(_const.Workdir, o.Workdir); err != nil {
-			return errors.WithStack(err)
+		if err := unstructured.SetNestedField(o.Config.Value(), o.Workdir, _const.Workdir); err != nil {
+			return errors.Wrapf(err, "failed to set %q to config", _const.Workdir)
 		}
 	}
 	if o.Artifact != "" {
 		// override artifact_file in config
-		if err := config.SetValue("artifact_file", o.Artifact); err != nil {
-			return errors.WithStack(err)
+		if err := unstructured.SetNestedField(o.Config.Value(), o.Artifact, "artifact_file"); err != nil {
+			return errors.Wrapf(err, "failed to set %q to config", "artifact_file")
 		}
 	}
 	for _, s := range o.Set {
@@ -242,7 +243,8 @@ func setValue(config *kkcorev1.Config, key, val string) error {
 			return errors.Wrapf(err, "failed to unmarshal json for %q", key)
 		}
 
-		return config.SetValue(key, value)
+		return errors.Wrapf(unstructured.SetNestedMap(config.Value(), value, key),
+			"failed to set %q to config", key)
 	case strings.HasPrefix(val, "[") && strings.HasSuffix(val, "]"):
 		var value []any
 		err := json.Unmarshal([]byte(val), &value)
@@ -250,13 +252,17 @@ func setValue(config *kkcorev1.Config, key, val string) error {
 			return errors.Wrapf(err, "failed to unmarshal json for %q", key)
 		}
 
-		return config.SetValue(key, value)
+		return errors.Wrapf(unstructured.SetNestedSlice(config.Value(), value, key),
+			"failed to set %q to config", key)
 	case strings.EqualFold(val, "TRUE") || strings.EqualFold(val, "YES") || strings.EqualFold(val, "Y"):
-		return config.SetValue(key, true)
+		return errors.Wrapf(unstructured.SetNestedField(config.Value(), true, key),
+			"failed to set %q to config", key)
 	case strings.EqualFold(val, "FALSE") || strings.EqualFold(val, "NO") || strings.EqualFold(val, "N"):
-		return config.SetValue(key, false)
+		return errors.Wrapf(unstructured.SetNestedField(config.Value(), false, key),
+			"failed to set %q to config", key)
 	default:
-		return config.SetValue(key, val)
+		return errors.Wrapf(unstructured.SetNestedField(config.Value(), val, key),
+			"failed to set %q to config", key)
 	}
 }
 

pkg/connector/ssh_connector.go

@@ -223,7 +223,7 @@ func (c *sshConnector) FetchFile(_ context.Context, src string, dst io.Writer) e
 
 // ExecuteCommand in remote host
 func (c *sshConnector) ExecuteCommand(_ context.Context, cmd string) ([]byte, error) {
-	cmd = fmt.Sprintf("sudo -SE %s << 'KUBEKEY_EOF'\n %s\nKUBEKEY_EOF\n", c.shell, cmd)
+	cmd = fmt.Sprintf("sudo -SE %s << 'KUBEKEY_EOF'\n%s\nKUBEKEY_EOF\n", c.shell, cmd)
 	klog.V(5).InfoS("exec ssh command", "cmd", cmd, "host", c.Host)
 	// create ssh session
 	session, err := c.client.NewSession()

pkg/const/helper.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 )
@@ -33,12 +34,8 @@ import (
 // If it fails to get the current working directory, it logs another informational message
 // and returns a default directory path "/opt/kubekey".
 func GetWorkdirFromConfig(config kkcorev1.Config) string {
-	workdir, err := config.GetValue(Workdir)
-	if err == nil {
-		wd, ok := workdir.(string)
-		if ok {
-			return wd
-		}
+	if wd, _, err := unstructured.NestedString(config.Value(), Workdir); err == nil {
+		return wd
 	}
 	klog.Info("work_dir is not set use current dir.")
 	wd, err := os.Getwd()

pkg/controllers/infrastructure/kkmachine_controller.go

@@ -12,6 +12,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -378,41 +379,41 @@ func (r *KKMachineReconciler) getConfig(scope *clusterScope, kkmachine *capkkinf
 		klog.InfoS("get default config", "config", config)
 	}
 
-	if err := config.SetValue(_const.Workdir, _const.CAPKKWorkdir); err != nil {
+	if err := unstructured.SetNestedField(config.Value(), _const.CAPKKWorkdir, _const.Workdir); err != nil {
 		return config, errors.Wrapf(err, "failed to set %q in config", _const.Workdir)
 	}
-	if err := config.SetValue("node_name", _const.ProviderID2Host(scope.Name, kkmachine.Spec.ProviderID)); err != nil {
-		return config, errors.Wrap(err, "failed to set \"node_name\" in config")
+	if err := unstructured.SetNestedField(config.Value(), _const.ProviderID2Host(scope.Name, kkmachine.Spec.ProviderID), "node_name"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "node_name")
 	}
-	if err := config.SetValue("kube_version", kkmachine.Spec.Version); err != nil {
-		return config, errors.Wrap(err, "failed to set \"kube_version\" in config")
+	if err := unstructured.SetNestedField(config.Value(), kkmachine.Spec.Version, "kube_version"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "kube_version")
 	}
-	if err := config.SetValue("kubernetes.cluster_name", scope.Cluster.Name); err != nil {
-		return config, errors.Wrap(err, "failed to set \"kubernetes.cluster_name\" in config")
+	if err := unstructured.SetNestedField(config.Value(), scope.Cluster.Name, "kubernetes", "cluster_name"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "kubernetes.cluster_name")
 	}
-	if err := config.SetValue("kubernetes.roles", kkmachine.Spec.Roles); err != nil {
-		return config, errors.Wrap(err, "failed to set \"kubernetes.roles\" in config")
+	if err := unstructured.SetNestedField(config.Value(), kkmachine.Spec.Roles, "kubernetes", "roles"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "kubernetes.roles")
 	}
-	if err := config.SetValue("cluster_network", scope.Cluster.Spec.ClusterNetwork); err != nil {
-		return config, errors.Wrap(err, "failed to set \"cluster_network\" in config")
+	if err := unstructured.SetNestedField(config.Value(), scope.Cluster.Spec.ClusterNetwork, "cluster_network"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "cluster_network")
 	}
 
 	switch scope.KKCluster.Spec.ControlPlaneEndpointType {
 	case capkkinfrav1beta1.ControlPlaneEndpointTypeVIP:
 		// should set vip addr to config
-		if err := config.SetValue("kubernetes.control_plane_endpoint.kube_vip.address", scope.Cluster.Spec.ControlPlaneEndpoint.Host); err != nil {
-			return config, errors.Wrap(err, "failed to set \"kubernetes.control_plane_endpoint.kube_vip.address\" in config")
+		if err := unstructured.SetNestedField(config.Value(), scope.Cluster.Spec.ControlPlaneEndpoint.Host, "kubernetes", "control_plane_endpoint", "kube_vip", "address"); err != nil {
+			return config, errors.Wrapf(err, "failed to set %q in config", "kubernetes.control_plane_endpoint.kube_vip.address")
 		}
 	case capkkinfrav1beta1.ControlPlaneEndpointTypeDNS:
 		// do nothing
 	default:
 		return config, errors.New("unsupport ControlPlaneEndpointType")
 	}
-	if err := config.SetValue("kubernetes.control_plane_endpoint.host", scope.Cluster.Spec.ControlPlaneEndpoint.Host); err != nil {
-		return config, errors.Wrap(err, "failed to set \"kubernetes.kube_vip.address\" in config")
+	if err := unstructured.SetNestedField(config.Value(), scope.Cluster.Spec.ControlPlaneEndpoint.Host, "kubernetes", "control_plane_endpoint", "host"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "kubernetes.control_plane_endpoint.host")
 	}
-	if err := config.SetValue("kubernetes.control_plane_endpoint.type", scope.KKCluster.Spec.ControlPlaneEndpointType); err != nil {
-		return config, errors.Wrap(err, "failed to set \"kubernetes.kube_vip.enabled\" in config")
+	if err := unstructured.SetNestedField(config.Value(), scope.KKCluster.Spec.ControlPlaneEndpointType, "kubernetes", "control_plane_endpoint", "type"); err != nil {
+		return config, errors.Wrapf(err, "failed to set %q in config", "kubernetes.control_plane_endpoint.kube_vip.type")
 	}
 
 	return config, nil

pkg/project/git.go

@@ -29,6 +29,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
 	kkprojectv1 "github.com/kubesphere/kubekey/api/project/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	_const "github.com/kubesphere/kubekey/v4/pkg/const"
 )
@@ -43,13 +44,9 @@ func newGitProject(ctx context.Context, playbook kkcorev1.Playbook, update bool)
 	}
 
 	// get project_dir from playbook
-	projectDir, err := playbook.Spec.Config.GetValue("project_dir")
+	projectDir, _, err := unstructured.NestedString(playbook.Spec.Config.Value(), _const.ProjectsDir)
 	if err != nil {
-		return nil, errors.Wrap(err, "project_dir is not defined")
-	}
-	pd, ok := projectDir.(string)
-	if !ok {
-		return nil, errors.New("project_dir should be string")
+		return nil, errors.Wrapf(err, "failed to get %q in config", _const.ProjectsDir)
 	}
 
 	// git clone to project dir
@@ -59,7 +56,7 @@ func newGitProject(ctx context.Context, playbook kkcorev1.Playbook, update bool)
 
 	p := &gitProject{
 		Playbook:   playbook,
-		projectDir: filepath.Join(pd, playbook.Spec.Project.Name),
+		projectDir: filepath.Join(projectDir, playbook.Spec.Project.Name),
 		playbook:   playbook.Spec.Playbook,
 	}
 

pkg/variable/variable_get_test.go

@@ -21,6 +21,7 @@ import (
 
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
 	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -232,6 +233,9 @@ func TestGetWorkdir(t *testing.T) {
 					Config: kkcorev1.Config{
 						Spec: runtime.RawExtension{
 							Raw: []byte("{\"work_dir\": \"abc\"}"),
+							Object: &unstructured.Unstructured{Object: map[string]any{
+								"work_dir": "abc",
+							}},
 						},
 					},
 				},