feat: add role dependency at meta (#2652)

Signed-off-by: joyceliu <joyceliu@yunify.com>

api/project/v1/playbook_test.go

@@ -97,7 +97,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) {
 					},
 					Roles: []Role{
 						{
-							RoleInfo{
+							RoleInfo: RoleInfo{
 								Role: "test",
 							},
 						},
@@ -121,7 +121,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) {
 					},
 					Roles: []Role{
 						{
-							RoleInfo{
+							RoleInfo: RoleInfo{
 								Role: "test",
 							},
 						},
@@ -146,7 +146,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) {
 					},
 					Roles: []Role{
 						{
-							RoleInfo{
+							RoleInfo: RoleInfo{
 								Conditional: Conditional{When: When{Data: []string{"true"}}},
 								Role:        "test",
 							},
@@ -172,7 +172,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) {
 					},
 					Roles: []Role{
 						{
-							RoleInfo{
+							RoleInfo: RoleInfo{
 								Conditional: Conditional{When: When{Data: []string{"true", "false"}}},
 								Role:        "test",
 							},

api/project/v1/role.go

@@ -32,6 +32,8 @@ type RoleInfo struct {
 	Taggable         `yaml:",inline"`
 	CollectionSearch `yaml:",inline"`
 
+	RoleDependency []Role `yaml:"dependencies,omitempty"`
+
 	// Role ref in playbook
 	Role string `yaml:"role,omitempty"`
 

pkg/const/workdir.go

@@ -24,6 +24,8 @@ work_dir/
 |   |   |-- playbooks/
 |   |   |-- roles/
 |   |   |   |-- roleName/
+|   |   |   |   |-- meta/
+|   |   |   |   |   |-- main.yml
 |   |   |   |   |-- tasks/
 |   |   |   |   |   |-- main.yml
 |   |   |   |   |-- defaults/
@@ -80,6 +82,9 @@ const ProjectRolesDir = "roles"
 
 // roleName represents the name of individual roles.
 
+// ProjectRolesMetaDir is a fixed directory name under a role, used to store meta information such as dependencies.
+const ProjectRolesMetaDir = "meta"
+
 // ProjectRolesTasksDir is a fixed directory name under a role, used to store tasks required by the role.
 const ProjectRolesTasksDir = "tasks"
 

pkg/executor/block_executor.go

@@ -39,7 +39,7 @@ func (e blockExecutor) Exec(ctx context.Context) error {
 		ignoreErrors := e.dealIgnoreErrors(block.IgnoreErrors)
 		when := e.dealWhen(block.When)
 
-		// // check tags
+		// check tags
 		if !tags.IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) {
 			// if not match the tags. skip
 			continue

pkg/executor/playbook_executor.go

@@ -173,25 +173,18 @@ func (e playbookExecutor) execBatchHosts(ctx context.Context, play kkprojectv1.P
 		}
 		// generate task from role
 		for _, role := range play.Roles {
-			if !kkprojectv1.JoinTag(role.Taggable, play.Taggable).IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) {
-				// if not match the tags. skip
-				continue
-			}
-			if err := e.variable.Merge(variable.MergeRuntimeVariable(role.Vars, serials...)); err != nil {
-				return err
-			}
 			// use the most closely configuration
 			ignoreErrors := role.IgnoreErrors
 			if ignoreErrors == nil {
 				ignoreErrors = play.IgnoreErrors
 			}
-			// role is block.
-			if err := (blockExecutor{
+
+			// role has block.
+			if err := (roleExecutor{
 				option:       e.option,
 				hosts:        serials,
 				ignoreErrors: ignoreErrors,
-				blocks:       role.Block,
-				role:         role.Role,
+				role:         role,
 				when:         role.When.Data,
 				tags:         kkprojectv1.JoinTag(role.Taggable, play.Taggable),
 			}.Exec(ctx)); err != nil {

pkg/executor/role_executor.go

@@ -0,0 +1,93 @@
+package executor
+
+import (
+	"context"
+	"slices"
+
+	kkprojectv1 "github.com/kubesphere/kubekey/api/project/v1"
+
+	"github.com/kubesphere/kubekey/v4/pkg/variable"
+)
+
+// roleExecutor is responsible for executing a role within a playbook.
+// It manages the execution of role dependencies, variable merging, and block execution.
+type roleExecutor struct {
+	*option
+
+	// playbook level config
+	hosts []string // which hosts will run playbook
+	// blocks level config
+	role         kkprojectv1.Role
+	ignoreErrors *bool // IgnoreErrors for role
+
+	when []string // when condition for merge
+	tags kkprojectv1.Taggable
+}
+
+// Exec executes the role, including its dependencies and blocks.
+// It checks tags, merges variables, and recursively executes dependent roles and blocks.
+func (e roleExecutor) Exec(ctx context.Context) error {
+	// check tags: skip execution if tags do not match
+	if !e.tags.IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) {
+		// if not match the tags. skip
+		return nil
+	}
+	// merge variables defined in the role for the current hosts
+	if err := e.variable.Merge(variable.MergeRuntimeVariable(e.role.Vars, e.hosts...)); err != nil {
+		return err
+	}
+	// deal dependency role: execute all role dependencies recursively
+	for _, dep := range e.role.RoleDependency {
+		// recursively execute the dependency role
+		if err := (roleExecutor{
+			option:       e.option,
+			role:         dep,
+			hosts:        e.hosts,
+			ignoreErrors: e.dealIgnoreErrors(dep.IgnoreErrors),
+			when:         e.dealWhen(dep.When),
+			tags:         e.dealTags(dep.Taggable),
+		}.Exec(ctx)); err != nil {
+			return err
+		}
+	}
+
+	// execute the blocks defined in the role
+	return (blockExecutor{
+		option:       e.option,
+		hosts:        e.hosts,
+		ignoreErrors: e.ignoreErrors,
+		blocks:       e.role.Block,
+		role:         e.role.Role,
+		when:         e.role.When.Data,
+		tags:         e.tags,
+	}.Exec(ctx))
+}
+
+// dealTags merges the provided taggable with the current tags.
+// "tags" argument in block. block tags inherits parent block.
+func (e roleExecutor) dealTags(taggable kkprojectv1.Taggable) kkprojectv1.Taggable {
+	return kkprojectv1.JoinTag(taggable, e.tags)
+}
+
+// dealIgnoreErrors returns the ignore_errors value for the block.
+// If ignore_errors is not defined in the block, it uses the value from the parent block.
+func (e roleExecutor) dealIgnoreErrors(ie *bool) *bool {
+	if ie == nil {
+		ie = e.ignoreErrors
+	}
+
+	return ie
+}
+
+// dealWhen merges the provided when conditions with the current ones.
+// Block when inherits parent block.
+func (e roleExecutor) dealWhen(when kkprojectv1.When) []string {
+	w := e.when
+	for _, d := range when.Data {
+		if !slices.Contains(w, d) {
+			w = append(w, d)
+		}
+	}
+
+	return w
+}

pkg/project/path.go

@@ -49,6 +49,18 @@ const (
 |-- [projectDir]
 |-- roles/
 |   |-- [role]/
+`
+
+	// PathFormatRoleMeta defines the directory structure for role meta information.
+	// The meta/main.yaml (or main.yml) file can contain dependencies on other roles via the "dependencies" key.
+	PathFormatRoleMeta = `
+|-- baseRole/
+|   |-- meta/
+|   |   |-- main.yaml
+
+|-- baseRole/
+|   |-- meta/
+|   |   |-- main.yml
 `
 
 	// PathFormatRoleTask defines the directory structure for role tasks
@@ -132,6 +144,15 @@ func GetRoleRelPath(basePlaybook string, role string) []string {
 	}
 }
 
+// GetRoleMetaRelPath returns possible relative paths for a role's meta file (main.yaml or main.yml) within the meta directory.
+// The format follows the standard Ansible role meta directory structure.
+func GetRoleMetaRelPath(baseRole string) []string {
+	return []string{
+		filepath.Join(baseRole, _const.ProjectRolesMetaDir, "main.yaml"),
+		filepath.Join(baseRole, _const.ProjectRolesMetaDir, "main.yml"),
+	}
+}
+
 // GetRoleTaskRelPath returns possible relative paths for a role's main task file
 // The format follows PathFormatRoleTask structure
 func GetRoleTaskRelPath(baseRole string) []string {

pkg/project/project.go

@@ -107,10 +107,6 @@ func (f *project) MarshalPlaybook() (*kkprojectv1.Playbook, error) {
 	if err := f.loadPlaybook(f.basePlaybook); err != nil {
 		return nil, err
 	}
-	// convertIncludeTasks
-	if err := f.convertIncludeTasks(f.basePlaybook); err != nil {
-		return nil, err
-	}
 	// validate playbook
 	if err := f.Playbook.Validate(); err != nil {
 		return nil, err
@@ -139,10 +135,29 @@ func (f *project) loadPlaybook(basePlaybook string) error {
 		if err := f.dealVarsFiles(&p, basePlaybook); err != nil {
 			return err
 		}
-		// fill block in roles
-		if err := f.dealRoles(p, basePlaybook); err != nil {
+		// deal "pre_tasks"
+		if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.PreTasks); err != nil {
 			return err
 		}
+		// deal "tasks"
+		if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.Tasks); err != nil {
+			return err
+		}
+		// deal "post_tasks"
+		if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.PostTasks); err != nil {
+			return err
+		}
+
+		//deal "roles"
+		for i := range p.Roles {
+			if err := f.dealRole(&p.Roles[i], basePlaybook); err != nil {
+				return err
+			}
+			// deal tasks
+			if err := f.dealRoleTask(&p.Roles[i], basePlaybook); err != nil {
+				return err
+			}
+		}
 
 		f.Playbook.Play = append(f.Playbook.Play, p)
 	}
@@ -195,18 +210,30 @@ func (f *project) dealVarsFiles(p *kkprojectv1.Play, basePlaybook string) error
 	return nil
 }
 
-// dealRoles handles the "roles" argument in a play
-func (f *project) dealRoles(p kkprojectv1.Play, basePlaybook string) error {
-	for i, r := range p.Roles {
-		baseRole := f.getPath(GetRoleRelPath(basePlaybook, r.Role))
-		if baseRole == "" {
-			return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", r.Role, basePlaybook, PathFormatRole)
+func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error {
+	baseRole := f.getPath(GetRoleRelPath(basePlaybook, role.Role))
+	if baseRole == "" {
+		return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", role.Role, basePlaybook, PathFormatRole)
+	}
+	// deal dependency
+	if meta := f.getPath(GetRoleMetaRelPath(baseRole)); meta != "" {
+		mdata, err := fs.ReadFile(f.FS, meta)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read role meta file %q", meta)
 		}
-		// deal tasks
-		task := f.getPath(GetRoleTaskRelPath(baseRole))
-		if task == "" {
-			return errors.Errorf("cannot found main task for Role %q. it's should be: \n %s", r.Role, PathFormatRoleTask)
+		roleMeta := &kkprojectv1.Role{}
+		if err := yaml.Unmarshal(mdata, roleMeta); err != nil {
+			return errors.Wrapf(err, "failed to unmarshal role meta file %q", meta)
 		}
+		for _, dep := range roleMeta.RoleDependency {
+			if err := f.dealRole(&dep, basePlaybook); err != nil {
+				return errors.Wrapf(err, "failed to deal dependency role base %q", role.Role)
+			}
+			role.RoleDependency = append(role.RoleDependency, dep)
+		}
+	}
+	// deal tasks
+	if task := f.getPath(GetRoleTaskRelPath(baseRole)); task != "" {
 		rdata, err := fs.ReadFile(f.FS, task)
 		if err != nil {
 			return errors.Wrapf(err, "failed to read file %q", task)
@@ -215,91 +242,85 @@ func (f *project) dealRoles(p kkprojectv1.Play, basePlaybook string) error {
 		if err := yaml.Unmarshal(rdata, &blocks); err != nil {
 			return errors.Wrapf(err, "failed to unmarshal yaml file %q", task)
 		}
-		p.Roles[i].Block = blocks
-		// deal defaults (optional)
-		if defaults := f.getPath(GetRoleDefaultsRelPath(baseRole)); defaults != "" {
-			data, err := fs.ReadFile(f.FS, defaults)
-			if err != nil {
-				return errors.Wrapf(err, "failed to read defaults variable file %q", defaults)
-			}
+		role.Block = blocks
+	}
+	// deal defaults (optional)
+	if defaults := f.getPath(GetRoleDefaultsRelPath(baseRole)); defaults != "" {
+		data, err := fs.ReadFile(f.FS, defaults)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read defaults variable file %q", defaults)
+		}
 
-			var node yaml.Node
-			// Unmarshal the YAML document into a root node.
-			if err := yaml.Unmarshal(data, &node); err != nil {
-				return errors.Wrap(err, "failed to unmarshal YAML")
-			}
-			if node.Kind != yaml.DocumentNode || len(node.Content) != 1 {
-				return errors.Errorf("unsupport vars_files format. it should be single map file")
-			}
-			// combine map node
-			if node.Content[0].Kind == yaml.MappingNode {
-				// skip empty file
-				p.Roles[i].Vars = *variable.CombineMappingNode(&p.Roles[i].Vars, node.Content[0])
-			}
+		var node yaml.Node
+		// Unmarshal the YAML document into a root node.
+		if err := yaml.Unmarshal(data, &node); err != nil {
+			return errors.Wrap(err, "failed to unmarshal YAML")
+		}
+		if node.Kind != yaml.DocumentNode || len(node.Content) != 1 {
+			return errors.Errorf("unsupport vars_files format. it should be single map file")
+		}
+		// combine map node
+		if node.Content[0].Kind == yaml.MappingNode {
+			// skip empty file
+			role.Vars = *variable.CombineMappingNode(&role.Vars, node.Content[0])
 		}
 	}
-
 	return nil
 }
 
-// convertIncludeTasks converts tasks from files into blocks
-func (f *project) convertIncludeTasks(basePlaybook string) error {
-	for _, play := range f.Playbook.Play {
-		if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.PreTasks); err != nil {
+// dealRoleTask recursively processes the tasks for a given role and its dependencies.
+// It ensures that all dependent roles are processed before handling the current role's tasks.
+func (f *project) dealRoleTask(role *kkprojectv1.Role, basePlaybook string) error {
+	for i := range role.RoleDependency {
+		if err := f.dealRoleTask(&role.RoleDependency[i], basePlaybook); err != nil {
 			return err
 		}
-		if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.Tasks); err != nil {
-			return err
-		}
-		if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.PostTasks); err != nil {
-			return err
-		}
-		for _, r := range play.Roles {
-			baseRole := f.getPath(GetRoleRelPath(basePlaybook, r.Role))
-			if baseRole == "" {
-				return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", r.Role, basePlaybook, PathFormatRole)
-			}
-			if err := f.fileToBlock(baseRole, filepath.Join(baseRole, _const.ProjectRolesTasksDir), r.Block); err != nil {
-				return err
-			}
-		}
 	}
-
-	return nil
+	// Get the base path for the current role
+	baseRole := f.getPath(GetRoleRelPath(basePlaybook, role.Role))
+	// Process the tasks for the current role
+	return f.dealBlock(baseRole, filepath.Join(baseRole, _const.ProjectRolesTasksDir), role.Block)
 }
 
-// fileToBlock converts task files into blocks, handling include_tasks directives
-func (f *project) fileToBlock(top string, source string, blocks []kkprojectv1.Block) error {
+// dealBlock recursively processes blocks, handling nested blocks, include_tasks, and annotating tasks with their relative path.
+func (f *project) dealBlock(top string, source string, blocks []kkprojectv1.Block) error {
 	for i, block := range blocks {
 		switch {
-		case len(block.Block) != 0: // it blocks
-			if err := f.fileToBlock(top, source, block.Block); err != nil {
+		case len(block.Block) != 0: // it's a block with nested blocks (block, rescue, always)
+			// Recursively process nested blocks
+			if err := f.dealBlock(top, source, block.Block); err != nil {
 				return err
 			}
-			if err := f.fileToBlock(top, source, block.Rescue); err != nil {
+			if err := f.dealBlock(top, source, block.Rescue); err != nil {
 				return err
 			}
-			if err := f.fileToBlock(top, source, block.Always); err != nil {
+			if err := f.dealBlock(top, source, block.Always); err != nil {
 				return err
 			}
-		case block.IncludeTasks != "": // it's include_tasks
+		case block.IncludeTasks != "": // it's an include_tasks directive
+			// Resolve the path to the include_tasks file
 			includeTask := f.getPath(GetIncludeTaskRelPath(top, source, block.IncludeTasks))
 			if includeTask == "" {
 				return errors.Errorf("failed to find include_task %q base on %q. it's should be:\n %s", block.IncludeTasks, source, PathFormatIncludeTask)
 			}
+			// Read the include_tasks file
 			data, err := fs.ReadFile(f.FS, includeTask)
 			if err != nil {
 				return errors.Wrapf(err, "failed to read includeTask file %q", includeTask)
 			}
+			// Unmarshal the file into blocks
 			var includeBlocks []kkprojectv1.Block
 			if err := yaml.Unmarshal(data, &includeBlocks); err != nil {
 				return errors.Wrapf(err, "failed to unmarshal includeTask file %q", includeTask)
 			}
-			if err := f.fileToBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil {
+			// Recursively process the included blocks
+			if err := f.dealBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil {
 				return err
 			}
+			// Assign the included blocks to the current block
 			blocks[i].Block = includeBlocks
-		default: // it tasks
+		default: // it's a regular task
+			// Annotate the task with its relative path
 			blocks[i].UnknownField["annotations"] = map[string]string{
 				kkcorev1alpha1.TaskAnnotationRelativePath: top,
 			}

pkg/project/project_test.go

@@ -510,6 +510,59 @@ func TestMarshalPlaybook(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "test_playbook5_dependency_role",
+			playbook: kkcorev1.Playbook{
+				Spec: kkcorev1.PlaybookSpec{
+					Playbook: "testdata/playbooks/playbook5.yaml",
+				},
+			},
+			except: &kkprojectv1.Playbook{
+				Play: []kkprojectv1.Play{
+					{
+						Base: kkprojectv1.Base{
+							Name: "dependency role",
+						},
+						PlayHost: kkprojectv1.PlayHost{
+							Hosts: []string{"node1"},
+						},
+						Roles: []kkprojectv1.Role{
+							{
+								RoleInfo: kkprojectv1.RoleInfo{
+									Role: "role3",
+									RoleDependency: []kkprojectv1.Role{
+										{
+											RoleInfo: kkprojectv1.RoleInfo{
+												Role: "role1",
+												Block: []kkprojectv1.Block{
+													{
+														BlockBase: kkprojectv1.BlockBase{
+															Base: kkprojectv1.Base{
+																Name: "task1",
+															},
+														},
+														Task: kkprojectv1.Task{
+															UnknownField: map[string]any{
+																"annotations": map[string]string{
+																	kkcorev1alpha1.TaskAnnotationRelativePath: "roles/role1",
+																},
+																"debug": map[string]any{
+																	"msg": "im task1",
+																},
+															},
+														},
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range testcases {

pkg/project/testdata/playbooks/playbook5.yaml

@@ -0,0 +1,4 @@
+- name: dependency role
+  hosts: node1
+  roles: 
+    - role3

pkg/project/testdata/roles/role3/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: role1