diff --git a/api/project/v1/playbook_test.go b/api/project/v1/playbook_test.go index d762d1df..8c4f5a48 100644 --- a/api/project/v1/playbook_test.go +++ b/api/project/v1/playbook_test.go @@ -97,7 +97,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) { }, Roles: []Role{ { - RoleInfo{ + RoleInfo: RoleInfo{ Role: "test", }, }, @@ -121,7 +121,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) { }, Roles: []Role{ { - RoleInfo{ + RoleInfo: RoleInfo{ Role: "test", }, }, @@ -146,7 +146,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) { }, Roles: []Role{ { - RoleInfo{ + RoleInfo: RoleInfo{ Conditional: Conditional{When: When{Data: []string{"true"}}}, Role: "test", }, @@ -172,7 +172,7 @@ func TestUnmarshalYamlPlaybook(t *testing.T) { }, Roles: []Role{ { - RoleInfo{ + RoleInfo: RoleInfo{ Conditional: Conditional{When: When{Data: []string{"true", "false"}}}, Role: "test", }, diff --git a/api/project/v1/role.go b/api/project/v1/role.go index 4a1815d7..68ad9c4d 100644 --- a/api/project/v1/role.go +++ b/api/project/v1/role.go @@ -32,6 +32,8 @@ type RoleInfo struct { Taggable `yaml:",inline"` CollectionSearch `yaml:",inline"` + RoleDependency []Role `yaml:"dependencies,omitempty"` + // Role ref in playbook Role string `yaml:"role,omitempty"` diff --git a/pkg/const/workdir.go b/pkg/const/workdir.go index d5255836..c225fe93 100644 --- a/pkg/const/workdir.go +++ b/pkg/const/workdir.go @@ -24,6 +24,8 @@ work_dir/ | | |-- playbooks/ | | |-- roles/ | | | |-- roleName/ +| | | | |-- meta/ +| | | | | |-- main.yml | | | | |-- tasks/ | | | | | |-- main.yml | | | | |-- defaults/ @@ -80,6 +82,9 @@ const ProjectRolesDir = "roles" // roleName represents the name of individual roles. +// ProjectRolesMetaDir is a fixed directory name under a role, used to store meta information such as dependencies. +const ProjectRolesMetaDir = "meta" + // ProjectRolesTasksDir is a fixed directory name under a role, used to store tasks required by the role. const ProjectRolesTasksDir = "tasks" diff --git a/pkg/executor/block_executor.go b/pkg/executor/block_executor.go index 3d294d90..b42f75d2 100644 --- a/pkg/executor/block_executor.go +++ b/pkg/executor/block_executor.go @@ -39,7 +39,7 @@ func (e blockExecutor) Exec(ctx context.Context) error { ignoreErrors := e.dealIgnoreErrors(block.IgnoreErrors) when := e.dealWhen(block.When) - // // check tags + // check tags if !tags.IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) { // if not match the tags. skip continue diff --git a/pkg/executor/playbook_executor.go b/pkg/executor/playbook_executor.go index 9833a433..e8ccea69 100644 --- a/pkg/executor/playbook_executor.go +++ b/pkg/executor/playbook_executor.go @@ -173,25 +173,18 @@ func (e playbookExecutor) execBatchHosts(ctx context.Context, play kkprojectv1.P } // generate task from role for _, role := range play.Roles { - if !kkprojectv1.JoinTag(role.Taggable, play.Taggable).IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) { - // if not match the tags. skip - continue - } - if err := e.variable.Merge(variable.MergeRuntimeVariable(role.Vars, serials...)); err != nil { - return err - } // use the most closely configuration ignoreErrors := role.IgnoreErrors if ignoreErrors == nil { ignoreErrors = play.IgnoreErrors } - // role is block. - if err := (blockExecutor{ + + // role has block. + if err := (roleExecutor{ option: e.option, hosts: serials, ignoreErrors: ignoreErrors, - blocks: role.Block, - role: role.Role, + role: role, when: role.When.Data, tags: kkprojectv1.JoinTag(role.Taggable, play.Taggable), }.Exec(ctx)); err != nil { diff --git a/pkg/executor/role_executor.go b/pkg/executor/role_executor.go new file mode 100644 index 00000000..11a8d003 --- /dev/null +++ b/pkg/executor/role_executor.go @@ -0,0 +1,93 @@ +package executor + +import ( + "context" + "slices" + + kkprojectv1 "github.com/kubesphere/kubekey/api/project/v1" + + "github.com/kubesphere/kubekey/v4/pkg/variable" +) + +// roleExecutor is responsible for executing a role within a playbook. +// It manages the execution of role dependencies, variable merging, and block execution. +type roleExecutor struct { + *option + + // playbook level config + hosts []string // which hosts will run playbook + // blocks level config + role kkprojectv1.Role + ignoreErrors *bool // IgnoreErrors for role + + when []string // when condition for merge + tags kkprojectv1.Taggable +} + +// Exec executes the role, including its dependencies and blocks. +// It checks tags, merges variables, and recursively executes dependent roles and blocks. +func (e roleExecutor) Exec(ctx context.Context) error { + // check tags: skip execution if tags do not match + if !e.tags.IsEnabled(e.playbook.Spec.Tags, e.playbook.Spec.SkipTags) { + // if not match the tags. skip + return nil + } + // merge variables defined in the role for the current hosts + if err := e.variable.Merge(variable.MergeRuntimeVariable(e.role.Vars, e.hosts...)); err != nil { + return err + } + // deal dependency role: execute all role dependencies recursively + for _, dep := range e.role.RoleDependency { + // recursively execute the dependency role + if err := (roleExecutor{ + option: e.option, + role: dep, + hosts: e.hosts, + ignoreErrors: e.dealIgnoreErrors(dep.IgnoreErrors), + when: e.dealWhen(dep.When), + tags: e.dealTags(dep.Taggable), + }.Exec(ctx)); err != nil { + return err + } + } + + // execute the blocks defined in the role + return (blockExecutor{ + option: e.option, + hosts: e.hosts, + ignoreErrors: e.ignoreErrors, + blocks: e.role.Block, + role: e.role.Role, + when: e.role.When.Data, + tags: e.tags, + }.Exec(ctx)) +} + +// dealTags merges the provided taggable with the current tags. +// "tags" argument in block. block tags inherits parent block. +func (e roleExecutor) dealTags(taggable kkprojectv1.Taggable) kkprojectv1.Taggable { + return kkprojectv1.JoinTag(taggable, e.tags) +} + +// dealIgnoreErrors returns the ignore_errors value for the block. +// If ignore_errors is not defined in the block, it uses the value from the parent block. +func (e roleExecutor) dealIgnoreErrors(ie *bool) *bool { + if ie == nil { + ie = e.ignoreErrors + } + + return ie +} + +// dealWhen merges the provided when conditions with the current ones. +// Block when inherits parent block. +func (e roleExecutor) dealWhen(when kkprojectv1.When) []string { + w := e.when + for _, d := range when.Data { + if !slices.Contains(w, d) { + w = append(w, d) + } + } + + return w +} diff --git a/pkg/project/path.go b/pkg/project/path.go index 7fb8e402..4cb44208 100644 --- a/pkg/project/path.go +++ b/pkg/project/path.go @@ -49,6 +49,18 @@ const ( |-- [projectDir] |-- roles/ | |-- [role]/ +` + + // PathFormatRoleMeta defines the directory structure for role meta information. + // The meta/main.yaml (or main.yml) file can contain dependencies on other roles via the "dependencies" key. + PathFormatRoleMeta = ` +|-- baseRole/ +| |-- meta/ +| | |-- main.yaml + +|-- baseRole/ +| |-- meta/ +| | |-- main.yml ` // PathFormatRoleTask defines the directory structure for role tasks @@ -132,6 +144,15 @@ func GetRoleRelPath(basePlaybook string, role string) []string { } } +// GetRoleMetaRelPath returns possible relative paths for a role's meta file (main.yaml or main.yml) within the meta directory. +// The format follows the standard Ansible role meta directory structure. +func GetRoleMetaRelPath(baseRole string) []string { + return []string{ + filepath.Join(baseRole, _const.ProjectRolesMetaDir, "main.yaml"), + filepath.Join(baseRole, _const.ProjectRolesMetaDir, "main.yml"), + } +} + // GetRoleTaskRelPath returns possible relative paths for a role's main task file // The format follows PathFormatRoleTask structure func GetRoleTaskRelPath(baseRole string) []string { diff --git a/pkg/project/project.go b/pkg/project/project.go index ad92bfad..f75fe73a 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -107,10 +107,6 @@ func (f *project) MarshalPlaybook() (*kkprojectv1.Playbook, error) { if err := f.loadPlaybook(f.basePlaybook); err != nil { return nil, err } - // convertIncludeTasks - if err := f.convertIncludeTasks(f.basePlaybook); err != nil { - return nil, err - } // validate playbook if err := f.Playbook.Validate(); err != nil { return nil, err @@ -139,10 +135,29 @@ func (f *project) loadPlaybook(basePlaybook string) error { if err := f.dealVarsFiles(&p, basePlaybook); err != nil { return err } - // fill block in roles - if err := f.dealRoles(p, basePlaybook); err != nil { + // deal "pre_tasks" + if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.PreTasks); err != nil { return err } + // deal "tasks" + if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.Tasks); err != nil { + return err + } + // deal "post_tasks" + if err := f.dealBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), p.PostTasks); err != nil { + return err + } + + //deal "roles" + for i := range p.Roles { + if err := f.dealRole(&p.Roles[i], basePlaybook); err != nil { + return err + } + // deal tasks + if err := f.dealRoleTask(&p.Roles[i], basePlaybook); err != nil { + return err + } + } f.Playbook.Play = append(f.Playbook.Play, p) } @@ -195,18 +210,30 @@ func (f *project) dealVarsFiles(p *kkprojectv1.Play, basePlaybook string) error return nil } -// dealRoles handles the "roles" argument in a play -func (f *project) dealRoles(p kkprojectv1.Play, basePlaybook string) error { - for i, r := range p.Roles { - baseRole := f.getPath(GetRoleRelPath(basePlaybook, r.Role)) - if baseRole == "" { - return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", r.Role, basePlaybook, PathFormatRole) +func (f *project) dealRole(role *kkprojectv1.Role, basePlaybook string) error { + baseRole := f.getPath(GetRoleRelPath(basePlaybook, role.Role)) + if baseRole == "" { + return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", role.Role, basePlaybook, PathFormatRole) + } + // deal dependency + if meta := f.getPath(GetRoleMetaRelPath(baseRole)); meta != "" { + mdata, err := fs.ReadFile(f.FS, meta) + if err != nil { + return errors.Wrapf(err, "failed to read role meta file %q", meta) } - // deal tasks - task := f.getPath(GetRoleTaskRelPath(baseRole)) - if task == "" { - return errors.Errorf("cannot found main task for Role %q. it's should be: \n %s", r.Role, PathFormatRoleTask) + roleMeta := &kkprojectv1.Role{} + if err := yaml.Unmarshal(mdata, roleMeta); err != nil { + return errors.Wrapf(err, "failed to unmarshal role meta file %q", meta) } + for _, dep := range roleMeta.RoleDependency { + if err := f.dealRole(&dep, basePlaybook); err != nil { + return errors.Wrapf(err, "failed to deal dependency role base %q", role.Role) + } + role.RoleDependency = append(role.RoleDependency, dep) + } + } + // deal tasks + if task := f.getPath(GetRoleTaskRelPath(baseRole)); task != "" { rdata, err := fs.ReadFile(f.FS, task) if err != nil { return errors.Wrapf(err, "failed to read file %q", task) @@ -215,91 +242,85 @@ func (f *project) dealRoles(p kkprojectv1.Play, basePlaybook string) error { if err := yaml.Unmarshal(rdata, &blocks); err != nil { return errors.Wrapf(err, "failed to unmarshal yaml file %q", task) } - p.Roles[i].Block = blocks - // deal defaults (optional) - if defaults := f.getPath(GetRoleDefaultsRelPath(baseRole)); defaults != "" { - data, err := fs.ReadFile(f.FS, defaults) - if err != nil { - return errors.Wrapf(err, "failed to read defaults variable file %q", defaults) - } + role.Block = blocks + } + // deal defaults (optional) + if defaults := f.getPath(GetRoleDefaultsRelPath(baseRole)); defaults != "" { + data, err := fs.ReadFile(f.FS, defaults) + if err != nil { + return errors.Wrapf(err, "failed to read defaults variable file %q", defaults) + } - var node yaml.Node - // Unmarshal the YAML document into a root node. - if err := yaml.Unmarshal(data, &node); err != nil { - return errors.Wrap(err, "failed to unmarshal YAML") - } - if node.Kind != yaml.DocumentNode || len(node.Content) != 1 { - return errors.Errorf("unsupport vars_files format. it should be single map file") - } - // combine map node - if node.Content[0].Kind == yaml.MappingNode { - // skip empty file - p.Roles[i].Vars = *variable.CombineMappingNode(&p.Roles[i].Vars, node.Content[0]) - } + var node yaml.Node + // Unmarshal the YAML document into a root node. + if err := yaml.Unmarshal(data, &node); err != nil { + return errors.Wrap(err, "failed to unmarshal YAML") + } + if node.Kind != yaml.DocumentNode || len(node.Content) != 1 { + return errors.Errorf("unsupport vars_files format. it should be single map file") + } + // combine map node + if node.Content[0].Kind == yaml.MappingNode { + // skip empty file + role.Vars = *variable.CombineMappingNode(&role.Vars, node.Content[0]) } } - return nil } -// convertIncludeTasks converts tasks from files into blocks -func (f *project) convertIncludeTasks(basePlaybook string) error { - for _, play := range f.Playbook.Play { - if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.PreTasks); err != nil { +// dealRoleTask recursively processes the tasks for a given role and its dependencies. +// It ensures that all dependent roles are processed before handling the current role's tasks. +func (f *project) dealRoleTask(role *kkprojectv1.Role, basePlaybook string) error { + for i := range role.RoleDependency { + if err := f.dealRoleTask(&role.RoleDependency[i], basePlaybook); err != nil { return err } - if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.Tasks); err != nil { - return err - } - if err := f.fileToBlock(filepath.Dir(basePlaybook), filepath.Dir(basePlaybook), play.PostTasks); err != nil { - return err - } - for _, r := range play.Roles { - baseRole := f.getPath(GetRoleRelPath(basePlaybook, r.Role)) - if baseRole == "" { - return errors.Errorf("failed to find role %q base on %q. it's should be:\n %s", r.Role, basePlaybook, PathFormatRole) - } - if err := f.fileToBlock(baseRole, filepath.Join(baseRole, _const.ProjectRolesTasksDir), r.Block); err != nil { - return err - } - } } - - return nil + // Get the base path for the current role + baseRole := f.getPath(GetRoleRelPath(basePlaybook, role.Role)) + // Process the tasks for the current role + return f.dealBlock(baseRole, filepath.Join(baseRole, _const.ProjectRolesTasksDir), role.Block) } -// fileToBlock converts task files into blocks, handling include_tasks directives -func (f *project) fileToBlock(top string, source string, blocks []kkprojectv1.Block) error { +// dealBlock recursively processes blocks, handling nested blocks, include_tasks, and annotating tasks with their relative path. +func (f *project) dealBlock(top string, source string, blocks []kkprojectv1.Block) error { for i, block := range blocks { switch { - case len(block.Block) != 0: // it blocks - if err := f.fileToBlock(top, source, block.Block); err != nil { + case len(block.Block) != 0: // it's a block with nested blocks (block, rescue, always) + // Recursively process nested blocks + if err := f.dealBlock(top, source, block.Block); err != nil { return err } - if err := f.fileToBlock(top, source, block.Rescue); err != nil { + if err := f.dealBlock(top, source, block.Rescue); err != nil { return err } - if err := f.fileToBlock(top, source, block.Always); err != nil { + if err := f.dealBlock(top, source, block.Always); err != nil { return err } - case block.IncludeTasks != "": // it's include_tasks + case block.IncludeTasks != "": // it's an include_tasks directive + // Resolve the path to the include_tasks file includeTask := f.getPath(GetIncludeTaskRelPath(top, source, block.IncludeTasks)) if includeTask == "" { return errors.Errorf("failed to find include_task %q base on %q. it's should be:\n %s", block.IncludeTasks, source, PathFormatIncludeTask) } + // Read the include_tasks file data, err := fs.ReadFile(f.FS, includeTask) if err != nil { return errors.Wrapf(err, "failed to read includeTask file %q", includeTask) } + // Unmarshal the file into blocks var includeBlocks []kkprojectv1.Block if err := yaml.Unmarshal(data, &includeBlocks); err != nil { return errors.Wrapf(err, "failed to unmarshal includeTask file %q", includeTask) } - if err := f.fileToBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil { + // Recursively process the included blocks + if err := f.dealBlock(top, filepath.Dir(includeTask), includeBlocks); err != nil { return err } + // Assign the included blocks to the current block blocks[i].Block = includeBlocks - default: // it tasks + default: // it's a regular task + // Annotate the task with its relative path blocks[i].UnknownField["annotations"] = map[string]string{ kkcorev1alpha1.TaskAnnotationRelativePath: top, } diff --git a/pkg/project/project_test.go b/pkg/project/project_test.go index 2893414c..7f7aeafc 100644 --- a/pkg/project/project_test.go +++ b/pkg/project/project_test.go @@ -510,6 +510,59 @@ func TestMarshalPlaybook(t *testing.T) { }, }, }, + { + name: "test_playbook5_dependency_role", + playbook: kkcorev1.Playbook{ + Spec: kkcorev1.PlaybookSpec{ + Playbook: "testdata/playbooks/playbook5.yaml", + }, + }, + except: &kkprojectv1.Playbook{ + Play: []kkprojectv1.Play{ + { + Base: kkprojectv1.Base{ + Name: "dependency role", + }, + PlayHost: kkprojectv1.PlayHost{ + Hosts: []string{"node1"}, + }, + Roles: []kkprojectv1.Role{ + { + RoleInfo: kkprojectv1.RoleInfo{ + Role: "role3", + RoleDependency: []kkprojectv1.Role{ + { + RoleInfo: kkprojectv1.RoleInfo{ + Role: "role1", + Block: []kkprojectv1.Block{ + { + BlockBase: kkprojectv1.BlockBase{ + Base: kkprojectv1.Base{ + Name: "task1", + }, + }, + Task: kkprojectv1.Task{ + UnknownField: map[string]any{ + "annotations": map[string]string{ + kkcorev1alpha1.TaskAnnotationRelativePath: "roles/role1", + }, + "debug": map[string]any{ + "msg": "im task1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range testcases { diff --git a/pkg/project/testdata/playbooks/playbook5.yaml b/pkg/project/testdata/playbooks/playbook5.yaml new file mode 100644 index 00000000..8157fe71 --- /dev/null +++ b/pkg/project/testdata/playbooks/playbook5.yaml @@ -0,0 +1,4 @@ +- name: dependency role + hosts: node1 + roles: + - role3 \ No newline at end of file diff --git a/pkg/project/testdata/roles/role3/meta/main.yaml b/pkg/project/testdata/roles/role3/meta/main.yaml new file mode 100644 index 00000000..1081b3c5 --- /dev/null +++ b/pkg/project/testdata/roles/role3/meta/main.yaml @@ -0,0 +1,2 @@ +dependencies: + - role: role1 \ No newline at end of file