/* Copyright 2023 The KubeSphere Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package core import ( "context" "github.com/cockroachdb/errors" kkcorev1 "github.com/kubesphere/kubekey/api/core/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlcontroller "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/kubesphere/kubekey/v4/cmd/controller-manager/app/options" _const "github.com/kubesphere/kubekey/v4/pkg/const" "github.com/kubesphere/kubekey/v4/pkg/controllers/util" ) const ( // playbookPodLabel set in pod. value is which playbook belongs to. podPlaybookLabel = "kubekey.kubesphere.io/playbook" executorContainer = "executor" ) // PlaybookReconciler reconcile playbook type PlaybookReconciler struct { ctrlclient.Client record.EventRecorder MaxConcurrentReconciles int } var _ options.Controller = &PlaybookReconciler{} var _ reconcile.Reconciler = &PlaybookReconciler{} // Name implements controllers.controller. func (r *PlaybookReconciler) Name() string { return "playbook-reconciler" } // SetupWithManager implements controllers.controller. func (r *PlaybookReconciler) SetupWithManager(mgr manager.Manager, o options.ControllerManagerServerOptions) error { r.Client = mgr.GetClient() r.EventRecorder = mgr.GetEventRecorderFor(r.Name()) return ctrl.NewControllerManagedBy(mgr). WithOptions(ctrlcontroller.Options{ MaxConcurrentReconciles: o.MaxConcurrentReconciles, }). For(&kkcorev1.Playbook{}). // Watches pod to sync playbook. Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj ctrlclient.Object) []reconcile.Request { playbook := &kkcorev1.Playbook{} if err := util.GetOwnerFromObject(ctx, r.Client, obj, playbook); err == nil { return []ctrl.Request{{NamespacedName: ctrlclient.ObjectKeyFromObject(playbook)}} } return nil })). Complete(r) } // +kubebuilder:rbac:groups=kubekey.kubesphere.io,resources=*,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=pods;events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;list;watch;create;update;patch;delete func (r PlaybookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { // get playbook playbook := &kkcorev1.Playbook{} if err := r.Client.Get(ctx, req.NamespacedName, playbook); err != nil { if !apierrors.IsNotFound(err) { return ctrl.Result{}, errors.Wrapf(err, "failed to get playbook %q", req.String()) } return ctrl.Result{}, nil } helper, err := patch.NewHelper(playbook, r.Client) if err != nil { return ctrl.Result{}, err } defer func() { if retErr != nil { if playbook.Status.FailureReason == "" { playbook.Status.FailureReason = kkcorev1.PlaybookFailedReasonUnknown } playbook.Status.FailureMessage = retErr.Error() } if err := r.reconcileStatus(ctx, playbook); err != nil { retErr = errors.Join(retErr, err) } if err := helper.Patch(ctx, playbook); err != nil { retErr = errors.Join(retErr, err) } }() // Add finalizer first if not set to avoid the race condition between init and delete. // Note: Finalizers in general can only be added when the deletionTimestamp is not set. if playbook.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(playbook, kkcorev1.PlaybookCompletedFinalizer) { controllerutil.AddFinalizer(playbook, kkcorev1.PlaybookCompletedFinalizer) return ctrl.Result{}, nil } // Handle deleted clusters if !playbook.DeletionTimestamp.IsZero() { return reconcile.Result{}, r.reconcileDelete(ctx, playbook) } return ctrl.Result{}, r.reconcileNormal(ctx, playbook) } func (r PlaybookReconciler) reconcileStatus(ctx context.Context, playbook *kkcorev1.Playbook) error { // get pod from playbook podList := &corev1.PodList{} if err := util.GetObjectListFromOwner(ctx, r.Client, playbook, podList); err != nil { return errors.Wrapf(err, "failed to get pod list from playbook %q", ctrlclient.ObjectKeyFromObject(playbook)) } // should only one pod for playbook if len(podList.Items) != 1 { return nil } if playbook.Status.Phase != kkcorev1.PlaybookPhaseFailed && playbook.Status.Phase != kkcorev1.PlaybookPhaseSucceeded { switch pod := podList.Items[0]; pod.Status.Phase { case corev1.PodFailed: playbook.Status.Phase = kkcorev1.PlaybookPhaseFailed playbook.Status.FailureReason = kkcorev1.PlaybookFailedReasonPodFailed playbook.Status.FailureMessage = pod.Status.Message for _, cs := range pod.Status.ContainerStatuses { if cs.Name == executorContainer && cs.State.Terminated != nil { playbook.Status.FailureMessage = cs.State.Terminated.Reason + ": " + cs.State.Terminated.Message } } case corev1.PodSucceeded: playbook.Status.Phase = kkcorev1.PlaybookPhaseSucceeded default: // the playbook status will set by pod } } return nil } func (r *PlaybookReconciler) reconcileDelete(ctx context.Context, playbook *kkcorev1.Playbook) error { podList := &corev1.PodList{} if err := util.GetObjectListFromOwner(ctx, r.Client, playbook, podList); err != nil { return err } if playbook.Status.Phase == kkcorev1.PlaybookPhaseFailed || playbook.Status.Phase == kkcorev1.PlaybookPhaseSucceeded { // playbook has completed. delete the owner pods. for _, obj := range podList.Items { if err := r.Client.Delete(ctx, &obj); err != nil { return errors.Wrapf(err, "failed to delete pod for %q", ctrlclient.ObjectKeyFromObject(playbook)) } } } if len(podList.Items) == 0 { controllerutil.RemoveFinalizer(playbook, kkcorev1.PlaybookCompletedFinalizer) } return nil } func (r *PlaybookReconciler) reconcileNormal(ctx context.Context, playbook *kkcorev1.Playbook) error { switch playbook.Status.Phase { case "": playbook.Status.Phase = kkcorev1.PlaybookPhasePending case kkcorev1.PlaybookPhasePending: playbook.Status.Phase = kkcorev1.PlaybookPhaseRunning case kkcorev1.PlaybookPhaseRunning: return r.dealRunningPlaybook(ctx, playbook) } return nil } func (r *PlaybookReconciler) dealRunningPlaybook(ctx context.Context, playbook *kkcorev1.Playbook) error { // check if pod is exist podList := &corev1.PodList{} if err := r.Client.List(ctx, podList, ctrlclient.InNamespace(playbook.Namespace), ctrlclient.MatchingLabels{ podPlaybookLabel: playbook.Name, }); err != nil && !apierrors.IsNotFound(err) { return errors.Wrapf(err, "failed to list pod with label %s=%s", podPlaybookLabel, playbook.Name) } else if len(podList.Items) != 0 { // could find exist pod return nil } // create pod pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: playbook.Name + "-", Namespace: playbook.Namespace, Labels: map[string]string{ podPlaybookLabel: playbook.Name, }, }, Spec: corev1.PodSpec{ ServiceAccountName: playbook.Spec.ServiceAccountName, RestartPolicy: "Never", Volumes: playbook.Spec.Volumes, Containers: []corev1.Container{ { Name: executorContainer, Image: _const.Getenv(_const.ExecutorImage), Command: []string{"kk"}, Args: []string{"playbook", "--name", playbook.Name, "--namespace", playbook.Namespace}, SecurityContext: &corev1.SecurityContext{ RunAsUser: ptr.To[int64](0), RunAsGroup: ptr.To[int64](0), }, VolumeMounts: playbook.Spec.VolumeMounts, }, }, }, } // get verbose from env if verbose := _const.Getenv(_const.ExecutorVerbose); verbose != "" { pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "-v", verbose) } // get ImagePullPolicy from env if imagePullPolicy := _const.Getenv(_const.ExecutorImagePullPolicy); imagePullPolicy != "" { pod.Spec.Containers[0].ImagePullPolicy = corev1.PullPolicy(imagePullPolicy) } if err := ctrl.SetControllerReference(playbook, pod, r.Client.Scheme()); err != nil { return errors.Wrapf(err, "failed to set ownerReference to playbook pod %q", pod.GenerateName) } return errors.Wrapf(r.Client.Create(ctx, pod), "failed to create pod of playbook %q", ctrlclient.ObjectKeyFromObject(playbook)) }