feat(capkk): add a kkinstance webhook to check if the in-place upgrade version is valid

Signed-off-by: 24sama <jacksama@foxmail.com>

PROJECT

@@ -66,4 +66,8 @@ resources:
   kind: KKInstance
   path: github.com/kubesphere/kubekey/api/v1beta1
   version: v1beta1
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 version: "3"

api/v1beta1/kkcluster_webhook.go

@@ -239,7 +239,7 @@ func validateInPlaceUpgrade(newAnnotation map[string]string) []*field.Error {
 			allErrs = append(allErrs,
 				field.InternalError(
 					field.NewPath("metadata", "annotations"),
-					errors.Wrapf(err, "failed to parse in-place upgrade version: %s", InPlaceUpgradeVersionAnnotation)),
+					errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)),
 			)
 		}
 	}

api/v1beta1/kkinstance_webhook.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2022 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"sigs.k8s.io/cluster-api/util/version"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+// log is for logging in this package.
+var kkinstancelog = logf.Log.WithName("kkinstance-resource")
+
+func (k *KKInstance) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(k).
+		Complete()
+}
+
+//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=default.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1
+
+var _ webhook.Defaulter = &KKInstance{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (k *KKInstance) Default() {
+	kkinstancelog.Info("default", "name", k.Name)
+}
+
+//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=validation.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1
+
+var _ webhook.Validator = &KKInstance{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (k *KKInstance) ValidateCreate() error {
+	kkinstancelog.Info("validate create", "name", k.Name)
+	return nil
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (k *KKInstance) ValidateUpdate(old runtime.Object) error {
+	kkinstancelog.Info("validate update", "name", k.Name)
+
+	var allErrs field.ErrorList
+	if v, ok := k.GetAnnotations()[InPlaceUpgradeVersionAnnotation]; ok {
+		if k.Status.NodeInfo == nil {
+			allErrs = append(allErrs,
+				field.Invalid(
+					field.NewPath("status", "nodeInfo"),
+					k.Status.NodeInfo,
+					"nodeInfo is required for in-place upgrade"))
+		}
+		newSemverVersion, err := version.ParseMajorMinorPatch(v)
+		if err != nil {
+			allErrs = append(allErrs,
+				field.InternalError(
+					field.NewPath("metadata", "annotations"),
+					errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)),
+			)
+		}
+		oldSemverVersion, err := version.ParseMajorMinorPatch(k.Status.NodeInfo.KubeletVersion)
+		if err != nil {
+			allErrs = append(allErrs,
+				field.InternalError(
+					field.NewPath("status", "nodeInfo", "kubeletVersion"),
+					errors.Wrapf(err, "failed to parse old version: %s", k.Status.NodeInfo.KubeletVersion)),
+			)
+		}
+
+		if newSemverVersion.Equals(oldSemverVersion) {
+			allErrs = append(allErrs,
+				field.Invalid(
+					field.NewPath("metadata", "annotations"),
+					v,
+					"new version must be different from old version"),
+			)
+		}
+
+		if !(newSemverVersion.GT(oldSemverVersion) &&
+			newSemverVersion.Major == oldSemverVersion.Major &&
+			(newSemverVersion.Minor == oldSemverVersion.Minor+1 || newSemverVersion.Minor == oldSemverVersion.Minor)) {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("metadata", "annotations"),
+					v, "Skipping MINOR versions when upgrading is unsupported."))
+		}
+	}
+
+	return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs)
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (k *KKInstance) ValidateDelete() error {
+	kkinstancelog.Info("validate delete", "name", k.Name)
+	return nil
+}

config/webhook/manifests.yaml

@@ -45,6 +45,26 @@ webhooks:
     resources:
     - kkclustertemplates
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance
+  failurePolicy: Fail
+  name: default.kkinstance.infrastructure.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - infrastructure.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - kkinstances
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:
@@ -132,6 +152,26 @@ webhooks:
     resources:
     - kkclustertemplates
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance
+  failurePolicy: Fail
+  name: validation.kkinstance.infrastructure.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - infrastructure.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - kkinstances
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

controllers/kkcluster/kkcluster_controller_phases.go

@@ -94,7 +94,9 @@ func (r *Reconciler) reconcileKKInstanceInPlaceUpgrade(ctx context.Context, clus
 				clusterScope.Error(err, "failed to patch annotation for kkInstance %s", kkiCopy.Name)
 				conditions.MarkFalse(kkCluster, infrav1.CallKKInstanceInPlaceUpgradeCondition, infrav1.KKInstanceObjectNotUpdatedReason,
 					clusterv1.ConditionSeverityWarning, "Failed to update annotation for kkInstance %s", kkiCopy.Name)
-				return ctrl.Result{}, err
+				r.Recorder.Eventf(kkCluster, corev1.EventTypeWarning, "FailedUpdateKKInstance",
+					"Failed to update kkInstance %s annotation: %v", kkiCopy.Name, err)
+				return ctrl.Result{RequeueAfter: 15 * time.Second}, err
 			}
 		}
 	}
@@ -155,7 +157,7 @@ func (r *Reconciler) upgradeChecks(_ context.Context, clusterScope *scope.Cluste
 	if len(kkInstanceErrors) > 0 {
 		aggregatedError := kerrors.NewAggregate(kkInstanceErrors)
 		r.Recorder.Eventf(clusterScope.KKCluster, corev1.EventTypeWarning, "KKInstanceInPlaceUpgradeUnFinished",
-			"Waiting for the KKInstance to pass upgrade checks to continue reconciliation: %v", aggregatedError)
+			"Waiting for all KKInstances to pass upgrade checks")
 		clusterScope.Info("Waiting for KKInstance to pass upgrade checks", "failures", aggregatedError.Error())
 
 		return ctrl.Result{RequeueAfter: upgradeCheckFailedRequeueAfter}, nil

main.go

@@ -167,6 +167,10 @@ func main() {
 		setupLog.Error(err, "unable to create webhook", "webhook", "KKMachineTemplate")
 		os.Exit(1)
 	}
+	if err = (&infrav1.KKInstance{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "KKInstance")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil {