Merge pull request #1010 from pixiake/registry

Support Kata and Node Feature Discovery

apis/kubekey/v1alpha2/kubernetes_types.go

@@ -18,6 +18,7 @@ package v1alpha2
 
 import "k8s.io/apimachinery/pkg/runtime"
 
+// Kubernetes contains the configuration for the cluster
 type Kubernetes struct {
 	Type                   string   `yaml:"type" json:"type,omitempty"`
 	Version                string   `yaml:"version" json:"version,omitempty"`
@@ -36,6 +37,8 @@ type Kubernetes struct {
 	EtcdBackupScriptDir      string               `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"`
 	ContainerManager         string               `yaml:"containerManager" json:"containerManager,omitempty"`
 	ContainerRuntimeEndpoint string               `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"`
+	NodeFeatureDiscovery     Kata                 `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"`
+	Kata                     NodeFeatureDiscovery `yaml:"kata" json:"kata,omitempty"`
 	ApiServerArgs            []string             `yaml:"apiserverArgs" json:"apiserverArgs,omitempty"`
 	ControllerManagerArgs    []string             `yaml:"controllerManagerArgs" json:"controllerManagerArgs,omitempty"`
 	SchedulerArgs            []string             `yaml:"schedulerArgs" json:"schedulerArgs,omitempty"`
@@ -46,6 +49,16 @@ type Kubernetes struct {
 	KubeProxyConfiguration   runtime.RawExtension `yaml:"kubeProxyConfiguration" json:"kubeProxyConfiguration,omitempty"`
 }
 
+// Kata contains the configuration for the kata in cluster
+type Kata struct {
+	Enabled *bool `yaml:"enabled" json:"enabled,omitempty"`
+}
+
+// NodeFeatureDiscovery contains the configuration for the node-feature-discovery in cluster
+type NodeFeatureDiscovery struct {
+	Enabled *bool `yaml:"enabled" json:"enabled,omitempty"`
+}
+
 // EnableNodelocaldns is used to determine whether to deploy nodelocaldns.
 func (k *Kubernetes) EnableNodelocaldns() bool {
 	if k.Nodelocaldns == nil {
@@ -53,3 +66,19 @@ func (k *Kubernetes) EnableNodelocaldns() bool {
 	}
 	return *k.Nodelocaldns
 }
+
+// EnableKataDeploy is used to determine whether to deploy kata.
+func (k *Kubernetes) EnableKataDeploy() bool {
+	if k.Kata.Enabled == nil {
+		return false
+	}
+	return *k.Kata.Enabled
+}
+
+// EnableNodeFeatureDiscovery is used to determine whether to deploy node-feature-discovery.
+func (k *Kubernetes) EnableNodeFeatureDiscovery() bool {
+	if k.NodeFeatureDiscovery.Enabled == nil {
+		return false
+	}
+	return *k.NodeFeatureDiscovery.Enabled
+}

apis/kubekey/v1alpha2/zz_generated.deepcopy.go

@@ -541,6 +541,26 @@ func (in *JobInfo) DeepCopy() *JobInfo {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Kata) DeepCopyInto(out *Kata) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kata.
+func (in *Kata) DeepCopy() *Kata {
+	if in == nil {
+		return nil
+	}
+	out := new(Kata)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeSphere) DeepCopyInto(out *KubeSphere) {
 	*out = *in
@@ -584,6 +604,8 @@ func (in *Kubernetes) DeepCopyInto(out *Kubernetes) {
 		*out = new(bool)
 		**out = **in
 	}
+	in.NodeFeatureDiscovery.DeepCopyInto(&out.NodeFeatureDiscovery)
+	in.Kata.DeepCopyInto(&out.Kata)
 	if in.ApiServerArgs != nil {
 		in, out := &in.ApiServerArgs, &out.ApiServerArgs
 		*out = make([]string, len(*in))
@@ -790,6 +812,26 @@ func (in *NetworkConfig) DeepCopy() *NetworkConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NodeFeatureDiscovery) DeepCopyInto(out *NodeFeatureDiscovery) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeFeatureDiscovery.
+func (in *NodeFeatureDiscovery) DeepCopy() *NodeFeatureDiscovery {
+	if in == nil {
+		return nil
+	}
+	out := new(NodeFeatureDiscovery)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NodeStatus) DeepCopyInto(out *NodeStatus) {
 	*out = *in

config/crd/bases/kubekey.kubesphere.io_clusters.yaml

@@ -463,6 +463,7 @@ spec:
                   type: object
                 type: array
               kubernetes:
+                description: Kubernetes contains the configuration for the cluster
                 properties:
                   apiserverArgs:
                     items:
@@ -494,6 +495,13 @@ spec:
                     additionalProperties:
                       type: boolean
                     type: object
+                  kata:
+                    description: NodeFeatureDiscovery contains the configuration for
+                      the node-feature-discovery in cluster
+                    properties:
+                      enabled:
+                        type: boolean
+                    type: object
                   keepBackupNumber:
                     type: integer
                   kubeProxyArgs:
@@ -514,6 +522,12 @@ spec:
                     type: integer
                   nodeCidrMaskSize:
                     type: integer
+                  nodeFeatureDiscovery:
+                    description: Kata contains the configuration for the kata in cluster
+                    properties:
+                      enabled:
+                        type: boolean
+                    type: object
                   nodelocaldns:
                     type: boolean
                   proxyMode:

pkg/images/tasks.go

@@ -118,9 +118,12 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
 		// storage
 		"provisioner-localpv": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "provisioner-localpv", Tag: "2.10.1", Group: kubekeyv1alpha2.Worker, Enable: false},
 		"linux-utils":         {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "linux-utils", Tag: "2.10.0", Group: kubekeyv1alpha2.Worker, Enable: false},
-
 		// load balancer
 		"haproxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "library", Repo: "haproxy", Tag: "2.3", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()},
+		// kata-deploy
+		"kata-deploy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kata-deploy", Tag: "stable", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.Kubernetes.EnableKataDeploy()},
+		// node-feature-discovery
+		"node-feature-discovery": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "node-feature-discovery", Tag: "v0.10.0", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery()},
 	}
 
 	image = ImageList[name]

pkg/pipelines/create_cluster.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kubesphere/kubekey/pkg/container"
 	"github.com/kubesphere/kubekey/pkg/images"
 	"github.com/kubesphere/kubekey/pkg/kubernetes"
+	"github.com/kubesphere/kubekey/pkg/plugins"
 	"github.com/kubesphere/kubekey/pkg/plugins/dns"
 	"io/ioutil"
 	"path/filepath"
@@ -77,6 +78,7 @@ func NewCreateClusterPipeline(runtime *common.KubeRuntime) error {
 		&filesystem.ChownModule{},
 		&certs.AutoRenewCertsModule{},
 		&kubernetes.SaveKubeConfigModule{},
+		&plugins.DeployPluginsModule{},
 		&addons.AddonsModule{},
 		&storage.DeployLocalVolumeModule{Skip: !runtime.Arg.DeployLocalStorage && !runtime.Cluster.KubeSphere.Enabled},
 		&kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled},

pkg/plugins/kata.go

@@ -0,0 +1,204 @@
+/*
+ Copyright 2022 The KubeSphere Authors.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package plugins
+
+import (
+	"github.com/kubesphere/kubekey/pkg/common"
+	"github.com/kubesphere/kubekey/pkg/core/action"
+	"github.com/kubesphere/kubekey/pkg/core/connector"
+	"github.com/kubesphere/kubekey/pkg/core/task"
+	"github.com/kubesphere/kubekey/pkg/core/util"
+	"github.com/kubesphere/kubekey/pkg/images"
+	"github.com/lithammer/dedent"
+	"github.com/pkg/errors"
+	"path/filepath"
+	"text/template"
+)
+
+// Kata Containers is an open source community working to build a secure container runtime with lightweight virtual
+// machines that feel and perform like containers, but provide stronger workload isolation using hardware virtualization
+// technology as a second layer of defense.
+
+var (
+	KataDeploy = template.Must(template.New("kata-deploy.yaml").Parse(
+		dedent.Dedent(`---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kata-label-node
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: node-labeler
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kata-label-node-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-labeler
+subjects:
+- kind: ServiceAccount
+  name: kata-label-node
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kata-deploy
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kata-deploy
+  template:
+    metadata:
+        labels:
+          name: kata-deploy
+    spec:
+      serviceAccountName: kata-label-node
+      containers:
+      - name: kube-kata
+        image: {{ .KataDeployImage }}
+        imagePullPolicy: Always
+        lifecycle:
+          preStop:
+            exec:
+              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: crio-conf
+          mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
+        - name: kata-artifacts
+          mountPath: /opt/kata/
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+        - name: local-bin
+          mountPath: /usr/local/bin/
+      volumes:
+        - name: crio-conf
+          hostPath:
+            path: /etc/crio/
+        - name: containerd-conf
+          hostPath:
+            path: /etc/containerd/
+        - name: kata-artifacts
+          hostPath:
+            path: /opt/kata/
+            type: DirectoryOrCreate
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin/
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-qemu
+handler: kata-qemu
+overhead:
+    podFixed:
+        memory: "160Mi"
+        cpu: "250m"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-clh
+handler: kata-clh
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-fc
+handler: kata-fc
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+    `)))
+)
+
+func DeployKataTasks(d *DeployPluginsModule) []task.Interface {
+	generateKataDeployManifests := &task.RemoteTask{
+		Name:    "GenerateKataDeployManifests",
+		Desc:    "Generate kata-deploy manifests",
+		Hosts:   d.Runtime.GetHostsByRole(common.Master),
+		Prepare: new(common.OnlyFirstMaster),
+		Action: &action.Template{
+			Template: KataDeploy,
+			Data: util.Data{
+				"KataDeployImage": images.GetImage(d.Runtime, d.KubeConf, "kata-deploy").ImageName(),
+			},
+			Dst: filepath.Join(common.KubeAddonsDir, KataDeploy.Name()),
+		},
+		Parallel: false,
+	}
+
+	deployKata := &task.RemoteTask{
+		Name:    "ApplyKataDeployManifests",
+		Desc:    "Apply kata-deploy manifests",
+		Hosts:   d.Runtime.GetHostsByRole(common.Master),
+		Prepare: new(common.OnlyFirstMaster),
+		Action:  new(ApplyKataDeployManifests),
+	}
+
+	return []task.Interface{
+		generateKataDeployManifests,
+		deployKata,
+	}
+}
+
+type ApplyKataDeployManifests struct {
+	common.KubeAction
+}
+
+func (a *ApplyKataDeployManifests) Execute(runtime connector.Runtime) error {
+	if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/kata-deploy.yaml", true); err != nil {
+		return errors.Wrap(errors.WithStack(err), "apply kata-deploy manifests failed")
+	}
+	return nil
+}

pkg/plugins/modules.go

@@ -0,0 +1,35 @@
+/*
+ Copyright 2022 The KubeSphere Authors.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package plugins
+
+import (
+	"github.com/kubesphere/kubekey/pkg/common"
+)
+
+type DeployPluginsModule struct {
+	common.KubeModule
+}
+
+func (d *DeployPluginsModule) Init() {
+	d.Name = "DeployPluginsModule"
+	d.Desc = "Deploy plugins for cluster"
+
+	if d.KubeConf.Cluster.Kubernetes.EnableKataDeploy() && (d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Conatinerd || d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Crio) {
+		d.Tasks = append(d.Tasks, DeployKataTasks(d)...)
+	}
+
+	if d.KubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery() {
+		d.Tasks = append(d.Tasks, DeployNodeFeatureDiscoveryTasks(d)...)
+	}
+}

pkg/plugins/nfd.go

@@ -0,0 +1,698 @@
+/*
+ Copyright 2022 The KubeSphere Authors.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package plugins
+
+import (
+	"github.com/kubesphere/kubekey/pkg/common"
+	"github.com/kubesphere/kubekey/pkg/core/action"
+	"github.com/kubesphere/kubekey/pkg/core/connector"
+	"github.com/kubesphere/kubekey/pkg/core/task"
+	"github.com/kubesphere/kubekey/pkg/core/util"
+	"github.com/kubesphere/kubekey/pkg/images"
+	"github.com/lithammer/dedent"
+	"github.com/pkg/errors"
+	"path/filepath"
+	"text/template"
+)
+
+// NodeFeatureDiscovery detects hardware features available on each node in a Kubernetes cluster, and advertises those
+// features using node labels.
+
+var (
+	NodeFeatureDiscovery = template.Must(template.New("node-feature-discovery.yaml").Parse(
+		dedent.Dedent(`---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: node-feature-discovery
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  name: nodefeaturerules.nfd.k8s-sigs.io
+spec:
+  group: nfd.k8s-sigs.io
+  names:
+    kind: NodeFeatureRule
+    listKind: NodeFeatureRuleList
+    plural: nodefeaturerules
+    singular: nodefeaturerule
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: NodeFeatureRule resource specifies a configuration for feature-based customization of node objects, such as node labeling.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: NodeFeatureRuleSpec describes a NodeFeatureRule.
+            properties:
+              rules:
+                description: Rules is a list of node customization rules.
+                items:
+                  description: Rule defines a rule for node customization such as labeling.
+                  properties:
+                    labels:
+                      additionalProperties:
+                        type: string
+                      description: Labels to create if the rule matches.
+                      type: object
+                    labelsTemplate:
+                      description: LabelsTemplate specifies a template to expand for dynamically generating multiple labels. Data (after template expansion) must be keys with an optional value (<key>[=<value>]) separated by newlines.
+                      type: string
+                    matchAny:
+                      description: MatchAny specifies a list of matchers one of which must match.
+                      items:
+                        description: MatchAnyElem specifies one sub-matcher of MatchAny.
+                        properties:
+                          matchFeatures:
+                            description: MatchFeatures specifies a set of matcher terms all of which must match.
+                            items:
+                              description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set.
+                              properties:
+                                feature:
+                                  type: string
+                                matchExpressions:
+                                  additionalProperties:
+                                    description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for     creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new     instance is created from scratch without using the helper functions."
+                                    properties:
+                                      op:
+                                        description: Op is the operator to be applied.
+                                        enum:
+                                        - In
+                                        - NotIn
+                                        - InRegexp
+                                        - Exists
+                                        - DoesNotExist
+                                        - Gt
+                                        - Lt
+                                        - GtLt
+                                        - IsTrue
+                                        - IsFalse
+                                        type: string
+                                      value:
+                                        description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - op
+                                    type: object
+                                  description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values.
+                                  type: object
+                              required:
+                              - feature
+                              - matchExpressions
+                              type: object
+                            type: array
+                        required:
+                        - matchFeatures
+                        type: object
+                      type: array
+                    matchFeatures:
+                      description: MatchFeatures specifies a set of matcher terms all of which must match.
+                      items:
+                        description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set.
+                        properties:
+                          feature:
+                            type: string
+                          matchExpressions:
+                            additionalProperties:
+                              description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for     creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new     instance is created from scratch without using the helper functions."
+                              properties:
+                                op:
+                                  description: Op is the operator to be applied.
+                                  enum:
+                                  - In
+                                  - NotIn
+                                  - InRegexp
+                                  - Exists
+                                  - DoesNotExist
+                                  - Gt
+                                  - Lt
+                                  - GtLt
+                                  - IsTrue
+                                  - IsFalse
+                                  type: string
+                                value:
+                                  description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element.
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                              - op
+                              type: object
+                            description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values.
+                            type: object
+                        required:
+                        - feature
+                        - matchExpressions
+                        type: object
+                      type: array
+                    name:
+                      description: Name of the rule.
+                      type: string
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: Vars is the variables to store if the rule matches. Variables do not directly inflict any changes in the node object. However, they can be referenced from other rules enabling more complex rule hierarchies, without exposing intermediary output values as labels.
+                      type: object
+                    varsTemplate:
+                      description: VarsTemplate specifies a template to expand for dynamically generating multiple variables. Data (after template expansion) must be keys with an optional value (<key>[=<value>]) separated by newlines.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+            required:
+            - rules
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nfd-master
+  namespace: node-feature-discovery
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: nfd-master
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - patch
+  - update
+  - list
+- apiGroups:
+  - topology.node.k8s.io
+  resources:
+  - noderesourcetopologies
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - nfd.k8s-sigs.io
+  resources:
+  - nodefeaturerules
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nfd-master
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nfd-master
+subjects:
+- kind: ServiceAccount
+  name: nfd-master
+  namespace: node-feature-discovery
+---
+apiVersion: v1
+data:
+  nfd-worker.conf: |
+    #core:
+    #  labelWhiteList:
+    #  noPublish: false
+    #  sleepInterval: 60s
+    #  featureSources: [all]
+    #  labelSources: [all]
+    #  klog:
+    #    addDirHeader: false
+    #    alsologtostderr: false
+    #    logBacktraceAt:
+    #    logtostderr: true
+    #    skipHeaders: false
+    #    stderrthreshold: 2
+    #    v: 0
+    #    vmodule:
+    ##   NOTE: the following options are not dynamically run-time configurable
+    ##         and require a nfd-worker restart to take effect after being changed
+    #    logDir:
+    #    logFile:
+    #    logFileMaxSize: 1800
+    #    skipLogHeaders: false
+    #sources:
+    #  cpu:
+    #    cpuid:
+    ##     NOTE: whitelist has priority over blacklist
+    #      attributeBlacklist:
+    #        - "BMI1"
+    #        - "BMI2"
+    #        - "CLMUL"
+    #        - "CMOV"
+    #        - "CX16"
+    #        - "ERMS"
+    #        - "F16C"
+    #        - "HTT"
+    #        - "LZCNT"
+    #        - "MMX"
+    #        - "MMXEXT"
+    #        - "NX"
+    #        - "POPCNT"
+    #        - "RDRAND"
+    #        - "RDSEED"
+    #        - "RDTSCP"
+    #        - "SGX"
+    #        - "SSE"
+    #        - "SSE2"
+    #        - "SSE3"
+    #        - "SSE4"
+    #        - "SSE42"
+    #        - "SSSE3"
+    #      attributeWhitelist:
+    #  kernel:
+    #    kconfigFile: "/path/to/kconfig"
+    #    configOpts:
+    #      - "NO_HZ"
+    #      - "X86"
+    #      - "DMI"
+    #  pci:
+    #    deviceClassWhitelist:
+    #      - "0200"
+    #      - "03"
+    #      - "12"
+    #    deviceLabelFields:
+    #      - "class"
+    #      - "vendor"
+    #      - "device"
+    #      - "subsystem_vendor"
+    #      - "subsystem_device"
+    #  usb:
+    #    deviceClassWhitelist:
+    #      - "0e"
+    #      - "ef"
+    #      - "fe"
+    #      - "ff"
+    #    deviceLabelFields:
+    #      - "class"
+    #      - "vendor"
+    #      - "device"
+    #  custom:
+    #    # The following feature demonstrates the capabilities of the matchFeatures
+    #    - name: "my custom rule"
+    #      labels:
+    #        my-ng-feature: "true"
+    #      # matchFeatures implements a logical AND over all matcher terms in the
+    #      # list (i.e. all of the terms, or per-feature matchers, must match)
+    #      matchFeatures:
+    #        - feature: cpu.cpuid
+    #          matchExpressions:
+    #            AVX512F: {op: Exists}
+    #        - feature: cpu.cstate
+    #          matchExpressions:
+    #            enabled: {op: IsTrue}
+    #        - feature: cpu.pstate
+    #          matchExpressions:
+    #            no_turbo: {op: IsFalse}
+    #            scaling_governor: {op: In, value: ["performance"]}
+    #        - feature: cpu.rdt
+    #          matchExpressions:
+    #            RDTL3CA: {op: Exists}
+    #        - feature: cpu.sst
+    #          matchExpressions:
+    #            bf.enabled: {op: IsTrue}
+    #        - feature: cpu.topology
+    #          matchExpressions:
+    #            hardware_multithreading: {op: IsFalse}
+    #
+    #        - feature: kernel.config
+    #          matchExpressions:
+    #            X86: {op: Exists}
+    #            LSM: {op: InRegexp, value: ["apparmor"]}
+    #        - feature: kernel.loadedmodule
+    #          matchExpressions:
+    #            e1000e: {op: Exists}
+    #        - feature: kernel.selinux
+    #          matchExpressions:
+    #            enabled: {op: IsFalse}
+    #        - feature: kernel.version
+    #          matchExpressions:
+    #            major: {op: In, value: ["5"]}
+    #            minor: {op: Gt, value: ["10"]}
+    #
+    #        - feature: storage.block
+    #          matchExpressions:
+    #            rotational: {op: In, value: ["0"]}
+    #            dax: {op: In, value: ["0"]}
+    #
+    #        - feature: network.device
+    #          matchExpressions:
+    #            operstate: {op: In, value: ["up"]}
+    #            speed: {op: Gt, value: ["100"]}
+    #
+    #        - feature: memory.numa
+    #          matchExpressions:
+    #            node_count: {op: Gt, value: ["2"]}
+    #        - feature: memory.nv
+    #          matchExpressions:
+    #            devtype: {op: In, value: ["nd_dax"]}
+    #            mode: {op: In, value: ["memory"]}
+    #
+    #        - feature: system.osrelease
+    #          matchExpressions:
+    #            ID: {op: In, value: ["fedora", "centos"]}
+    #        - feature: system.name
+    #          matchExpressions:
+    #            nodename: {op: InRegexp, value: ["^worker-X"]}
+    #
+    #        - feature: local.label
+    #          matchExpressions:
+    #            custom-feature-knob: {op: Gt, value: ["100"]}
+    #
+    #    # The following feature demonstrates the capabilities of the matchAny
+    #    - name: "my matchAny rule"
+    #      labels:
+    #        my-ng-feature-2: "my-value"
+    #      # matchAny implements a logical IF over all elements (sub-matchers) in
+    #      # the list (i.e. at least one feature matcher must match)
+    #      matchAny:
+    #        - matchFeatures:
+    #            - feature: kernel.loadedmodule
+    #              matchExpressions:
+    #                driver-module-X: {op: Exists}
+    #            - feature: pci.device
+    #              matchExpressions:
+    #                vendor: {op: In, value: ["8086"]}
+    #                class: {op: In, value: ["0200"]}
+    #        - matchFeatures:
+    #            - feature: kernel.loadedmodule
+    #              matchExpressions:
+    #                driver-module-Y: {op: Exists}
+    #            - feature: usb.device
+    #              matchExpressions:
+    #                vendor: {op: In, value: ["8086"]}
+    #                class: {op: In, value: ["02"]}
+    #
+    #    # The following features demonstreate label templating capabilities
+    #    - name: "my template rule"
+    #      labelsTemplate: |
+    #      matchFeatures:
+    #        - feature: system.osrelease
+    #          matchExpressions:
+    #            ID: {op: InRegexp, value: ["^open.*"]}
+    #            VERSION_ID.major: {op: In, value: ["13", "15"]}
+    #
+    #    - name: "my template rule 2"
+    #      matchFeatures:
+    #        - feature: pci.device
+    #          matchExpressions:
+    #            class: {op: InRegexp, value: ["^06"]}
+    #            vendor: ["8086"]
+    #        - feature: cpu.cpuid
+    #          matchExpressions:
+    #            AVX: {op: Exists}
+    #
+    #    # The following examples demonstrate vars field and back-referencing
+    #    # previous labels and vars
+    #    - name: "my dummy kernel rule"
+    #      labels:
+    #        "my.kernel.feature": "true"
+    #      matchFeatures:
+    #        - feature: kernel.version
+    #          matchExpressions:
+    #            major: {op: Gt, value: ["2"]}
+    #
+    #    - name: "my dummy rule with no labels"
+    #      vars:
+    #        "my.dummy.var": "1"
+    #      matchFeatures:
+    #        - feature: cpu.cpuid
+    #          matchExpressions: {}
+    #
+    #    - name: "my rule using backrefs"
+    #      labels:
+    #        "my.backref.feature": "true"
+    #      matchFeatures:
+    #        - feature: rule.matched
+    #          matchExpressions:
+    #            my.kernel.feature: {op: IsTrue}
+    #            my.dummy.var: {op: Gt, value: ["0"]}
+    #
+kind: ConfigMap
+metadata:
+  name: nfd-worker-conf
+  namespace: node-feature-discovery
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nfd-master
+  namespace: node-feature-discovery
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+  selector:
+    app: nfd-master
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nfd
+  name: nfd-master
+  namespace: node-feature-discovery
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nfd-master
+  template:
+    metadata:
+      labels:
+        app: nfd-master
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - ""
+            weight: 1
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: In
+                values:
+                - ""
+            weight: 1
+      containers:
+      - args: []
+        command:
+        - nfd-master
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: {{ .NFDImage }}
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /usr/bin/grpc_health_probe
+            - -addr=:8080
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        name: nfd-master
+        readinessProbe:
+          exec:
+            command:
+            - /usr/bin/grpc_health_probe
+            - -addr=:8080
+          failureThreshold: 10
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        volumeMounts: []
+      serviceAccount: nfd-master
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal
+        value: ""
+      volumes: []
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: nfd
+  name: nfd-worker
+  namespace: node-feature-discovery
+spec:
+  selector:
+    matchLabels:
+      app: nfd-worker
+  template:
+    metadata:
+      labels:
+        app: nfd-worker
+    spec:
+      containers:
+      - args:
+        - -server=nfd-master:8080
+        command:
+        - nfd-worker
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: {{ .NFDImage }}
+        imagePullPolicy: IfNotPresent
+        name: nfd-worker
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        volumeMounts:
+        - mountPath: /host-boot
+          name: host-boot
+          readOnly: true
+        - mountPath: /host-etc/os-release
+          name: host-os-release
+          readOnly: true
+        - mountPath: /host-sys
+          name: host-sys
+          readOnly: true
+        - mountPath: /host-usr/lib
+          name: host-usr-lib
+          readOnly: true
+        - mountPath: /etc/kubernetes/node-feature-discovery/source.d/
+          name: source-d
+          readOnly: true
+        - mountPath: /etc/kubernetes/node-feature-discovery/features.d/
+          name: features-d
+          readOnly: true
+        - mountPath: /etc/kubernetes/node-feature-discovery
+          name: nfd-worker-conf
+          readOnly: true
+      dnsPolicy: ClusterFirstWithHostNet
+      volumes:
+      - hostPath:
+          path: /boot
+        name: host-boot
+      - hostPath:
+          path: /etc/os-release
+        name: host-os-release
+      - hostPath:
+          path: /sys
+        name: host-sys
+      - hostPath:
+          path: /usr/lib
+        name: host-usr-lib
+      - hostPath:
+          path: /etc/kubernetes/node-feature-discovery/source.d/
+        name: source-d
+      - hostPath:
+          path: /etc/kubernetes/node-feature-discovery/features.d/
+        name: features-d
+      - configMap:
+          name: nfd-worker-conf
+        name: nfd-worker-conf
+
+    `)))
+)
+
+func DeployNodeFeatureDiscoveryTasks(d *DeployPluginsModule) []task.Interface {
+	generateNFDManifests := &task.RemoteTask{
+		Name:    "GenerateNFDManifests",
+		Desc:    "Generate node-feature-discovery manifests",
+		Hosts:   d.Runtime.GetHostsByRole(common.Master),
+		Prepare: new(common.OnlyFirstMaster),
+		Action: &action.Template{
+			Template: NodeFeatureDiscovery,
+			Data: util.Data{
+				"NFDImage": images.GetImage(d.Runtime, d.KubeConf, "node-feature-discovery").ImageName(),
+			},
+			Dst: filepath.Join(common.KubeAddonsDir, NodeFeatureDiscovery.Name()),
+		},
+		Parallel: false,
+	}
+
+	deployNFD := &task.RemoteTask{
+		Name:    "ApplyNFDManifests",
+		Desc:    "Apply node-feature-discovery manifests",
+		Hosts:   d.Runtime.GetHostsByRole(common.Master),
+		Prepare: new(common.OnlyFirstMaster),
+		Action:  new(ApplyNodeFeatureDiscoveryManifests),
+	}
+
+	return []task.Interface{
+		generateNFDManifests,
+		deployNFD,
+	}
+}
+
+type ApplyNodeFeatureDiscoveryManifests struct {
+	common.KubeAction
+}
+
+func (a *ApplyNodeFeatureDiscoveryManifests) Execute(runtime connector.Runtime) error {
+	if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/node-feature-discovery.yaml", true); err != nil {
+		return errors.Wrap(errors.WithStack(err), "apply node-feature-discovery manifests failed")
+	}
+	return nil
+}