feature: support same key in different file (#2714)

feature: support same key in different file



feature: support same key in different file



feature: support same key in different file



feature: support same key in different file

Signed-off-by: xuesongzuo@yunify.com <xuesongzuo@yunify.com>

api/project/v1/base.go

@@ -28,7 +28,8 @@ type Base struct {
 	RemoteUser string `yaml:"remote_user,omitempty"`
 
 	// variables
-	Vars yaml.Node `yaml:"vars,omitempty"`
+	Vars            []yaml.Node `yaml:"-"`
+	VarsFromMarshal yaml.Node   `yaml:"vars,omitempty"`
 
 	// module default params
 	//ModuleDefaults []map[string]map[string]any `yaml:"module_defaults,omitempty"`

pkg/executor/task_executor.go

@@ -267,7 +267,7 @@ func (e *taskExecutor) executeModule(ctx context.Context, task *kkcorev1alpha1.T
 		}
 
 		// Merge item into host's runtime variables
-		if err := e.variable.Merge(variable.MergeRuntimeVariable(node, host)); err != nil {
+		if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, host)); err != nil {
 			return errors.Wrap(err, "failed to set loop item to variable")
 		}
 
@@ -282,7 +282,7 @@ func (e *taskExecutor) executeModule(ctx context.Context, task *kkcorev1alpha1.T
 				resErr = errors.Wrap(err, "failed to convert loop item")
 				return
 			}
-			if err := e.variable.Merge(variable.MergeRuntimeVariable(resetNode, host)); err != nil {
+			if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{resetNode}, host)); err != nil {
 				resErr = errors.Wrap(err, "failed to clean loop item to variable")
 				return
 			}
@@ -400,7 +400,7 @@ func (e *taskExecutor) dealRegister(host string, stdout, stderr, errMsg string)
 		if err != nil {
 			return err
 		}
-		if err := e.variable.Merge(variable.MergeRuntimeVariable(node, host)); err != nil {
+		if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, host)); err != nil {
 			return err
 		}
 	}

pkg/modules/set_fact.go

@@ -66,7 +66,7 @@ func ModuleSetFact(_ context.Context, options ExecOptions) (string, string, erro
 	if err := yaml.Unmarshal(options.Args.Raw, &node); err != nil {
 		return StdoutFailed, "failed to unmarshal YAML", err
 	}
-	if err := options.Variable.Merge(variable.MergeRuntimeVariable(node, options.Host)); err != nil {
+	if err := options.Variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, options.Host)); err != nil {
 		return StdoutFailed, "failed to merge set_fact variable", err
 	}
 

pkg/project/project.go

@@ -34,7 +34,6 @@ import (
 	_const "github.com/kubesphere/kubekey/v4/pkg/const"
 	"github.com/kubesphere/kubekey/v4/pkg/converter/tmpl"
 	"github.com/kubesphere/kubekey/v4/pkg/utils"
-	"github.com/kubesphere/kubekey/v4/pkg/variable"
 )
 
 // builtinProjectFunc is a function that creates a Project from a built-in playbook
@@ -130,6 +129,10 @@ func (f *project) loadPlaybook(basePlaybook string) error {
 	}
 
 	for _, p := range plays {
+		if !p.VarsFromMarshal.IsZero() {
+			p.Vars = append(p.Vars, p.VarsFromMarshal)
+		}
+
 		if err := f.dealImportPlaybook(p, basePlaybook); err != nil {
 			return err
 		}
@@ -209,7 +212,7 @@ func (f *project) dealVarsFiles(p *kkprojectv1.Play, basePlaybook string) error
 		// combine map node
 		if node.Content[0].Kind == yaml.MappingNode {
 			// skip empty file
-			p.Vars = *variable.CombineMappingNode(&p.Vars, node.Content[0])
+			p.Vars = append(p.Vars, *node.Content[0])
 		}
 	}
 
@@ -231,6 +234,9 @@ func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error {
 		if err := yaml.Unmarshal(mdata, roleMeta); err != nil {
 			return errors.Wrapf(err, "failed to unmarshal role meta file %q", meta)
 		}
+		if !roleMeta.VarsFromMarshal.IsZero() {
+			roleMeta.Vars = append(roleMeta.Vars, roleMeta.VarsFromMarshal)
+		}
 		for _, dep := range roleMeta.RoleDependency {
 			if err := f.dealRole(&dep, basePlaybook); err != nil {
 				return errors.Wrapf(err, "failed to deal dependency role base %q", role.Role)
@@ -248,6 +254,11 @@ func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error {
 		if err := yaml.Unmarshal(rdata, &blocks); err != nil {
 			return errors.Wrapf(err, "failed to unmarshal yaml file %q", task)
 		}
+		for i, b := range blocks {
+			if !b.VarsFromMarshal.IsZero() {
+				blocks[i].Vars = append(b.Vars, b.VarsFromMarshal)
+			}
+		}
 		role.Block = blocks
 	}
 	// deal defaults (optional)
@@ -288,7 +299,7 @@ func (f *project) combineRoleVars(role *kkprojectv1.Role, content []byte) error
 	// combine map node
 	if node.Content[0].Kind == yaml.MappingNode {
 		// skip empty file
-		role.Vars = *variable.CombineMappingNode(&role.Vars, node.Content[0])
+		role.Vars = append(role.Vars, *node.Content[0])
 	}
 	return nil
 }
@@ -338,6 +349,11 @@ func (f *project) dealBlock(top string, source string, blocks []kkprojectv1.Bloc
 			if err := yaml.Unmarshal(data, &includeBlocks); err != nil {
 				return errors.Wrapf(err, "failed to unmarshal includeTask file %q", includeTask)
 			}
+			for i, b := range includeBlocks {
+				if !b.VarsFromMarshal.IsZero() {
+					includeBlocks[i].Vars = append(b.Vars, b.VarsFromMarshal)
+				}
+			}
 			// Recursively process the included blocks
 			if err := f.dealBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil {
 				return err

pkg/project/project_test.go

@@ -308,7 +308,7 @@ func TestMarshalPlaybook(t *testing.T) {
 					{
 						Base: kkprojectv1.Base{
 							Name: "playbook-var1",
-							Vars: yaml.Node{
+							VarsFromMarshal: yaml.Node{
 								Kind:   yaml.MappingNode,
 								Tag:    "!!map",
 								Line:   6,
@@ -330,6 +330,30 @@ func TestMarshalPlaybook(t *testing.T) {
 									},
 								},
 							},
+							Vars: []yaml.Node{
+								{
+									Kind:   yaml.MappingNode,
+									Tag:    "!!map",
+									Line:   6,
+									Column: 5,
+									Content: []*yaml.Node{
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a",
+											Line:   6,
+											Column: 5,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "b",
+											Line:   6,
+											Column: 8,
+										},
+									},
+								},
+							},
 						},
 						PlayHost: kkprojectv1.PlayHost{
 							Hosts: []string{"node1"},
@@ -351,81 +375,90 @@ func TestMarshalPlaybook(t *testing.T) {
 						VarsFiles: []string{"vars/var1.yaml", "vars/var2.yaml"},
 						Base: kkprojectv1.Base{
 							Name: "playbook-var2",
-							Vars: yaml.Node{
-								Kind:   yaml.MappingNode,
-								Tag:    "!!map",
-								Line:   2,
-								Column: 1,
-								Content: []*yaml.Node{
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a1",
-										Line:   2,
-										Column: 1,
+							Vars: []yaml.Node{
+								{
+									Kind:   yaml.MappingNode,
+									Tag:    "!!map",
+									Line:   2,
+									Column: 1,
+									Content: []*yaml.Node{
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a1",
+											Line:   2,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "aa",
+											Line:   2,
+											Column: 5,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a2",
+											Line:   3,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!int",
+											Value:  "1",
+											Line:   3,
+											Column: 5,
+										},
 									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "aa",
-										Line:   2,
-										Column: 5,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a2",
-										Line:   3,
-										Column: 1,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!int",
-										Value:  "1",
-										Line:   3,
-										Column: 5,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a2",
-										Line:   1,
-										Column: 1,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "aaa",
-										Line:   1,
-										Column: 5,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a3",
-										Line:   2,
-										Column: 1,
-									},
-									{
-										Kind:   yaml.MappingNode,
-										Tag:    "!!map",
-										Value:  "",
-										Line:   3,
-										Column: 2,
-										Content: []*yaml.Node{
-											{
-												Kind:   yaml.ScalarNode,
-												Tag:    "!!str",
-												Value:  "b3",
-												Line:   3,
-												Column: 2,
-											},
-											{
-												Kind:   yaml.ScalarNode,
-												Tag:    "!!int",
-												Value:  "1",
-												Line:   3,
-												Column: 6,
+								}, {
+									Kind:   yaml.MappingNode,
+									Tag:    "!!map",
+									Line:   1,
+									Column: 1,
+									Content: []*yaml.Node{
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a2",
+											Line:   1,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "aaa",
+											Line:   1,
+											Column: 5,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a3",
+											Line:   2,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.MappingNode,
+											Tag:    "!!map",
+											Value:  "",
+											Line:   3,
+											Column: 2,
+											Content: []*yaml.Node{
+												{
+													Kind:   yaml.ScalarNode,
+													Tag:    "!!str",
+													Value:  "b3",
+													Line:   3,
+													Column: 2,
+												},
+												{
+													Kind:   yaml.ScalarNode,
+													Tag:    "!!int",
+													Value:  "1",
+													Line:   3,
+													Column: 6,
+												},
 											},
 										},
 									},
@@ -452,7 +485,7 @@ func TestMarshalPlaybook(t *testing.T) {
 						VarsFiles: []string{"vars/var1.yaml"},
 						Base: kkprojectv1.Base{
 							Name: "playbook-var3",
-							Vars: yaml.Node{
+							VarsFromMarshal: yaml.Node{
 								Kind:   yaml.MappingNode,
 								Tag:    "!!map",
 								Line:   8,
@@ -472,33 +505,64 @@ func TestMarshalPlaybook(t *testing.T) {
 										Line:   8,
 										Column: 9,
 									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a1",
-										Line:   2,
-										Column: 1,
+								},
+							},
+							Vars: []yaml.Node{
+								{
+									Kind:   yaml.MappingNode,
+									Tag:    "!!map",
+									Line:   8,
+									Column: 5,
+									Content: []*yaml.Node{
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a2",
+											Line:   8,
+											Column: 5,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!int",
+											Value:  "2",
+											Line:   8,
+											Column: 9,
+										},
 									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "aa",
-										Line:   2,
-										Column: 5,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!str",
-										Value:  "a2",
-										Line:   3,
-										Column: 1,
-									},
-									{
-										Kind:   yaml.ScalarNode,
-										Tag:    "!!int",
-										Value:  "1",
-										Line:   3,
-										Column: 5,
+								}, {
+									Kind:   yaml.MappingNode,
+									Tag:    "!!map",
+									Line:   2,
+									Column: 1,
+									Content: []*yaml.Node{
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a1",
+											Line:   2,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "aa",
+											Line:   2,
+											Column: 5,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!str",
+											Value:  "a2",
+											Line:   3,
+											Column: 1,
+										},
+										{
+											Kind:   yaml.ScalarNode,
+											Tag:    "!!int",
+											Value:  "1",
+											Line:   3,
+											Column: 5,
+										},
 									},
 								},
 							},
@@ -575,7 +639,7 @@ func TestMarshalPlaybook(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			assert.Equal(t, tc.except, actual)
+			assert.Equal(t, tc.except, actual, tc.name)
 		})
 	}
 }

pkg/variable/helper.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/cockroachdb/errors"
 	kkcorev1 "github.com/kubesphere/kubekey/api/core/v1"
-	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/json"
@@ -36,29 +35,6 @@ import (
 	"github.com/kubesphere/kubekey/v4/pkg/converter/tmpl"
 )
 
-// CombineMappingNode combines two yaml.Node objects representing mapping nodes.
-// If a is nil or zero, returns b.
-// If both a and b are mapping nodes, appends b's content to a.
-// Returns a in all other cases.
-//
-// Parameters:
-//   - a: First yaml.Node to combine
-//   - b: Second yaml.Node to combine
-//
-// Returns:
-//   - Combined yaml.Node, with b taking precedence
-func CombineMappingNode(a, b *yaml.Node) *yaml.Node {
-	if a == nil || a.IsZero() {
-		return b
-	}
-
-	if a.Kind == yaml.MappingNode && b.Kind == yaml.MappingNode {
-		a.Content = append(a.Content, b.Content...)
-	}
-
-	return a
-}
-
 // CombineVariables merge multiple variables into one variable.
 // It recursively combines two maps, where values from m2 override values from m1 if keys overlap.
 // For nested maps, it will recursively merge their contents.

pkg/variable/variable_merge.go

@@ -36,8 +36,8 @@ var MergeRemoteVariable = func(data map[string]any, hostnames ...string) MergeFu
 // 1. Gets all variables for the host to create a parsing context
 // 2. Parses the YAML node using that context
 // 3. Merges the parsed data into the host's RuntimeVars
-var MergeRuntimeVariable = func(node yaml.Node, hosts ...string) MergeFunc {
-	if node.IsZero() {
+var MergeRuntimeVariable = func(nodes []yaml.Node, hosts ...string) MergeFunc {
+	if len(nodes) == 0 {
 		// skip
 		return emptyMergeFunc
 	}
@@ -58,13 +58,18 @@ var MergeRuntimeVariable = func(node yaml.Node, hosts ...string) MergeFunc {
 			if !ok {
 				return errors.Errorf("host %s variables type error, expect map[string]any", hostname)
 			}
-			data, err := parseYamlNode(ctx, node)
-			if err != nil {
-				return err
+			for _, node := range nodes {
+				if node.IsZero() {
+					continue
+				}
+				data, err := parseYamlNode(ctx, node)
+				if err != nil {
+					return err
+				}
+				hv := vv.value.Hosts[hostname]
+				hv.RuntimeVars = CombineVariables(hv.RuntimeVars, data)
+				vv.value.Hosts[hostname] = hv
 			}
-			hv := vv.value.Hosts[hostname]
-			hv.RuntimeVars = CombineVariables(hv.RuntimeVars, data)
-			vv.value.Hosts[hostname] = hv
 		}
 
 		return nil

pkg/variable/variable_merge_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
 
 	"github.com/kubesphere/kubekey/v4/pkg/converter"
 	"github.com/kubesphere/kubekey/v4/pkg/variable/source"
@@ -99,7 +100,7 @@ func TestMergeRuntimeVariable(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			if err := tc.variable.Merge(MergeRuntimeVariable(node, tc.hostname)); err != nil {
+			if err := tc.variable.Merge(MergeRuntimeVariable([]yaml.Node{node}, tc.hostname)); err != nil {
 				t.Fatal(err)
 			}