From aaae2f66346408eb064b6545b40b2c30b90d71da Mon Sep 17 00:00:00 2001 From: zuoxuesong-worker Date: Wed, 20 Aug 2025 11:10:55 +0800 Subject: [PATCH] feature: support same key in different file (#2714) feature: support same key in different file feature: support same key in different file feature: support same key in different file feature: support same key in different file Signed-off-by: xuesongzuo@yunify.com --- api/project/v1/base.go | 3 +- pkg/executor/task_executor.go | 6 +- pkg/modules/set_fact.go | 2 +- pkg/project/project.go | 22 ++- pkg/project/project_test.go | 270 +++++++++++++++++----------- pkg/variable/helper.go | 24 --- pkg/variable/variable_merge.go | 21 ++- pkg/variable/variable_merge_test.go | 3 +- 8 files changed, 207 insertions(+), 144 deletions(-) diff --git a/api/project/v1/base.go b/api/project/v1/base.go index 3f39994b..a33d88f7 100644 --- a/api/project/v1/base.go +++ b/api/project/v1/base.go @@ -28,7 +28,8 @@ type Base struct { RemoteUser string `yaml:"remote_user,omitempty"` // variables - Vars yaml.Node `yaml:"vars,omitempty"` + Vars []yaml.Node `yaml:"-"` + VarsFromMarshal yaml.Node `yaml:"vars,omitempty"` // module default params //ModuleDefaults []map[string]map[string]any `yaml:"module_defaults,omitempty"` diff --git a/pkg/executor/task_executor.go b/pkg/executor/task_executor.go index c2b1c373..ca7e7dee 100644 --- a/pkg/executor/task_executor.go +++ b/pkg/executor/task_executor.go @@ -267,7 +267,7 @@ func (e *taskExecutor) executeModule(ctx context.Context, task *kkcorev1alpha1.T } // Merge item into host's runtime variables - if err := e.variable.Merge(variable.MergeRuntimeVariable(node, host)); err != nil { + if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, host)); err != nil { return errors.Wrap(err, "failed to set loop item to variable") } @@ -282,7 +282,7 @@ func (e *taskExecutor) executeModule(ctx context.Context, task *kkcorev1alpha1.T resErr = errors.Wrap(err, "failed to convert loop item") return } - if err := e.variable.Merge(variable.MergeRuntimeVariable(resetNode, host)); err != nil { + if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{resetNode}, host)); err != nil { resErr = errors.Wrap(err, "failed to clean loop item to variable") return } @@ -400,7 +400,7 @@ func (e *taskExecutor) dealRegister(host string, stdout, stderr, errMsg string) if err != nil { return err } - if err := e.variable.Merge(variable.MergeRuntimeVariable(node, host)); err != nil { + if err := e.variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, host)); err != nil { return err } } diff --git a/pkg/modules/set_fact.go b/pkg/modules/set_fact.go index 1b875982..452def56 100644 --- a/pkg/modules/set_fact.go +++ b/pkg/modules/set_fact.go @@ -66,7 +66,7 @@ func ModuleSetFact(_ context.Context, options ExecOptions) (string, string, erro if err := yaml.Unmarshal(options.Args.Raw, &node); err != nil { return StdoutFailed, "failed to unmarshal YAML", err } - if err := options.Variable.Merge(variable.MergeRuntimeVariable(node, options.Host)); err != nil { + if err := options.Variable.Merge(variable.MergeRuntimeVariable([]yaml.Node{node}, options.Host)); err != nil { return StdoutFailed, "failed to merge set_fact variable", err } diff --git a/pkg/project/project.go b/pkg/project/project.go index f9828ece..334f0b3d 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -34,7 +34,6 @@ import ( _const "github.com/kubesphere/kubekey/v4/pkg/const" "github.com/kubesphere/kubekey/v4/pkg/converter/tmpl" "github.com/kubesphere/kubekey/v4/pkg/utils" - "github.com/kubesphere/kubekey/v4/pkg/variable" ) // builtinProjectFunc is a function that creates a Project from a built-in playbook @@ -130,6 +129,10 @@ func (f *project) loadPlaybook(basePlaybook string) error { } for _, p := range plays { + if !p.VarsFromMarshal.IsZero() { + p.Vars = append(p.Vars, p.VarsFromMarshal) + } + if err := f.dealImportPlaybook(p, basePlaybook); err != nil { return err } @@ -209,7 +212,7 @@ func (f *project) dealVarsFiles(p *kkprojectv1.Play, basePlaybook string) error // combine map node if node.Content[0].Kind == yaml.MappingNode { // skip empty file - p.Vars = *variable.CombineMappingNode(&p.Vars, node.Content[0]) + p.Vars = append(p.Vars, *node.Content[0]) } } @@ -231,6 +234,9 @@ func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error { if err := yaml.Unmarshal(mdata, roleMeta); err != nil { return errors.Wrapf(err, "failed to unmarshal role meta file %q", meta) } + if !roleMeta.VarsFromMarshal.IsZero() { + roleMeta.Vars = append(roleMeta.Vars, roleMeta.VarsFromMarshal) + } for _, dep := range roleMeta.RoleDependency { if err := f.dealRole(&dep, basePlaybook); err != nil { return errors.Wrapf(err, "failed to deal dependency role base %q", role.Role) @@ -248,6 +254,11 @@ func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error { if err := yaml.Unmarshal(rdata, &blocks); err != nil { return errors.Wrapf(err, "failed to unmarshal yaml file %q", task) } + for i, b := range blocks { + if !b.VarsFromMarshal.IsZero() { + blocks[i].Vars = append(b.Vars, b.VarsFromMarshal) + } + } role.Block = blocks } // deal defaults (optional) @@ -288,7 +299,7 @@ func (f *project) combineRoleVars(role *kkprojectv1.Role, content []byte) error // combine map node if node.Content[0].Kind == yaml.MappingNode { // skip empty file - role.Vars = *variable.CombineMappingNode(&role.Vars, node.Content[0]) + role.Vars = append(role.Vars, *node.Content[0]) } return nil } @@ -338,6 +349,11 @@ func (f *project) dealBlock(top string, source string, blocks []kkprojectv1.Bloc if err := yaml.Unmarshal(data, &includeBlocks); err != nil { return errors.Wrapf(err, "failed to unmarshal includeTask file %q", includeTask) } + for i, b := range includeBlocks { + if !b.VarsFromMarshal.IsZero() { + includeBlocks[i].Vars = append(b.Vars, b.VarsFromMarshal) + } + } // Recursively process the included blocks if err := f.dealBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil { return err diff --git a/pkg/project/project_test.go b/pkg/project/project_test.go index 7f7aeafc..c22b6a8b 100644 --- a/pkg/project/project_test.go +++ b/pkg/project/project_test.go @@ -308,7 +308,7 @@ func TestMarshalPlaybook(t *testing.T) { { Base: kkprojectv1.Base{ Name: "playbook-var1", - Vars: yaml.Node{ + VarsFromMarshal: yaml.Node{ Kind: yaml.MappingNode, Tag: "!!map", Line: 6, @@ -330,6 +330,30 @@ func TestMarshalPlaybook(t *testing.T) { }, }, }, + Vars: []yaml.Node{ + { + Kind: yaml.MappingNode, + Tag: "!!map", + Line: 6, + Column: 5, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a", + Line: 6, + Column: 5, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "b", + Line: 6, + Column: 8, + }, + }, + }, + }, }, PlayHost: kkprojectv1.PlayHost{ Hosts: []string{"node1"}, @@ -351,81 +375,90 @@ func TestMarshalPlaybook(t *testing.T) { VarsFiles: []string{"vars/var1.yaml", "vars/var2.yaml"}, Base: kkprojectv1.Base{ Name: "playbook-var2", - Vars: yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Line: 2, - Column: 1, - Content: []*yaml.Node{ - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a1", - Line: 2, - Column: 1, + Vars: []yaml.Node{ + { + Kind: yaml.MappingNode, + Tag: "!!map", + Line: 2, + Column: 1, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a1", + Line: 2, + Column: 1, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "aa", + Line: 2, + Column: 5, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a2", + Line: 3, + Column: 1, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "1", + Line: 3, + Column: 5, + }, }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "aa", - Line: 2, - Column: 5, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a2", - Line: 3, - Column: 1, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "1", - Line: 3, - Column: 5, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a2", - Line: 1, - Column: 1, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "aaa", - Line: 1, - Column: 5, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a3", - Line: 2, - Column: 1, - }, - { - Kind: yaml.MappingNode, - Tag: "!!map", - Value: "", - Line: 3, - Column: 2, - Content: []*yaml.Node{ - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "b3", - Line: 3, - Column: 2, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "1", - Line: 3, - Column: 6, + }, { + Kind: yaml.MappingNode, + Tag: "!!map", + Line: 1, + Column: 1, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a2", + Line: 1, + Column: 1, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "aaa", + Line: 1, + Column: 5, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a3", + Line: 2, + Column: 1, + }, + { + Kind: yaml.MappingNode, + Tag: "!!map", + Value: "", + Line: 3, + Column: 2, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "b3", + Line: 3, + Column: 2, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "1", + Line: 3, + Column: 6, + }, }, }, }, @@ -452,7 +485,7 @@ func TestMarshalPlaybook(t *testing.T) { VarsFiles: []string{"vars/var1.yaml"}, Base: kkprojectv1.Base{ Name: "playbook-var3", - Vars: yaml.Node{ + VarsFromMarshal: yaml.Node{ Kind: yaml.MappingNode, Tag: "!!map", Line: 8, @@ -472,33 +505,64 @@ func TestMarshalPlaybook(t *testing.T) { Line: 8, Column: 9, }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a1", - Line: 2, - Column: 1, + }, + }, + Vars: []yaml.Node{ + { + Kind: yaml.MappingNode, + Tag: "!!map", + Line: 8, + Column: 5, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a2", + Line: 8, + Column: 5, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "2", + Line: 8, + Column: 9, + }, }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "aa", - Line: 2, - Column: 5, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "a2", - Line: 3, - Column: 1, - }, - { - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "1", - Line: 3, - Column: 5, + }, { + Kind: yaml.MappingNode, + Tag: "!!map", + Line: 2, + Column: 1, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a1", + Line: 2, + Column: 1, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "aa", + Line: 2, + Column: 5, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "a2", + Line: 3, + Column: 1, + }, + { + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: "1", + Line: 3, + Column: 5, + }, }, }, }, @@ -575,7 +639,7 @@ func TestMarshalPlaybook(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, tc.except, actual) + assert.Equal(t, tc.except, actual, tc.name) }) } } diff --git a/pkg/variable/helper.go b/pkg/variable/helper.go index e8e8a31e..eecdc399 100644 --- a/pkg/variable/helper.go +++ b/pkg/variable/helper.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/errors" kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" - "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" @@ -36,29 +35,6 @@ import ( "github.com/kubesphere/kubekey/v4/pkg/converter/tmpl" ) -// CombineMappingNode combines two yaml.Node objects representing mapping nodes. -// If a is nil or zero, returns b. -// If both a and b are mapping nodes, appends b's content to a. -// Returns a in all other cases. -// -// Parameters: -// - a: First yaml.Node to combine -// - b: Second yaml.Node to combine -// -// Returns: -// - Combined yaml.Node, with b taking precedence -func CombineMappingNode(a, b *yaml.Node) *yaml.Node { - if a == nil || a.IsZero() { - return b - } - - if a.Kind == yaml.MappingNode && b.Kind == yaml.MappingNode { - a.Content = append(a.Content, b.Content...) - } - - return a -} - // CombineVariables merge multiple variables into one variable. // It recursively combines two maps, where values from m2 override values from m1 if keys overlap. // For nested maps, it will recursively merge their contents. diff --git a/pkg/variable/variable_merge.go b/pkg/variable/variable_merge.go index a2962cb0..1f7babe8 100644 --- a/pkg/variable/variable_merge.go +++ b/pkg/variable/variable_merge.go @@ -36,8 +36,8 @@ var MergeRemoteVariable = func(data map[string]any, hostnames ...string) MergeFu // 1. Gets all variables for the host to create a parsing context // 2. Parses the YAML node using that context // 3. Merges the parsed data into the host's RuntimeVars -var MergeRuntimeVariable = func(node yaml.Node, hosts ...string) MergeFunc { - if node.IsZero() { +var MergeRuntimeVariable = func(nodes []yaml.Node, hosts ...string) MergeFunc { + if len(nodes) == 0 { // skip return emptyMergeFunc } @@ -58,13 +58,18 @@ var MergeRuntimeVariable = func(node yaml.Node, hosts ...string) MergeFunc { if !ok { return errors.Errorf("host %s variables type error, expect map[string]any", hostname) } - data, err := parseYamlNode(ctx, node) - if err != nil { - return err + for _, node := range nodes { + if node.IsZero() { + continue + } + data, err := parseYamlNode(ctx, node) + if err != nil { + return err + } + hv := vv.value.Hosts[hostname] + hv.RuntimeVars = CombineVariables(hv.RuntimeVars, data) + vv.value.Hosts[hostname] = hv } - hv := vv.value.Hosts[hostname] - hv.RuntimeVars = CombineVariables(hv.RuntimeVars, data) - vv.value.Hosts[hostname] = hv } return nil diff --git a/pkg/variable/variable_merge_test.go b/pkg/variable/variable_merge_test.go index 004ce324..98d4e241 100644 --- a/pkg/variable/variable_merge_test.go +++ b/pkg/variable/variable_merge_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" "github.com/kubesphere/kubekey/v4/pkg/converter" "github.com/kubesphere/kubekey/v4/pkg/variable/source" @@ -99,7 +100,7 @@ func TestMergeRuntimeVariable(t *testing.T) { if err != nil { t.Fatal(err) } - if err := tc.variable.Merge(MergeRuntimeVariable(node, tc.hostname)); err != nil { + if err := tc.variable.Merge(MergeRuntimeVariable([]yaml.Node{node}, tc.hostname)); err != nil { t.Fatal(err) }