diff --git a/apis/v1alpha1/cluster.go b/apis/v1alpha1/cluster.go index e4ad7e43..e9dcf1e9 100644 --- a/apis/v1alpha1/cluster.go +++ b/apis/v1alpha1/cluster.go @@ -1,31 +1,50 @@ package v1alpha1 +import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + DefaultPreDir = "kubekey" + DefaultSSHPort = "22" + DefaultDockerSockPath = "/var/run/docker.sock" + DefaultLBPort = "6443" + DefaultLBDomain = "lb.kubesphere.local" + DefaultNetworkPlugin = "calico" + DefaultPodsCIDR = "10.233.64.0/18" + DefaultServiceCIDR = "10.233.0.0/18" + DefaultKubeImageRepo = "kubekey" + DefaultClusterName = "cluster.local" + DefaultArch = "amd64" + DefaultHostName = "allinone" + DefaultEtcdRepo = "kubekey/etcd" + DefaultEtcdVersion = "v3.3.12" + DefaultEtcdPort = "2379" + DefaultKubeVersion = "v1.17.4" + DefaultCniVersion = "v0.8.2" + DefaultHelmVersion = "v3.1.2" + ETCDRole = "etcd" + MasterRole = "master" + WorkerRole = "worker" +) + type ClusterCfg struct { Hosts []HostCfg `yaml:"hosts" json:"hosts,omitempty"` - LBKubeApiserver LBKubeApiserverCfg `yaml:"lb_kubeapiserver" json:"lb_kubeapiserver,omitempty"` - KubeVersion string `yaml:"kube_version" json:"kube_version,omitempty"` - KubeImageRepo string `yaml:"kube_image_repo" json:"kube_image_repo,omitempty"` - KubeClusterName string `yaml:"kube_cluster_name" json:"kube_cluster_name,omitempty"` + LBKubeApiserver LBKubeApiserverCfg `yaml:"lbKubeapiserver" json:"lbKubeapiserver,omitempty"` + KubeVersion string `yaml:"kubeVersion" json:"kubeVersion,omitempty"` + KubeImageRepo string `yaml:"kubeImageRepo" json:"kubeImageRepo,omitempty"` + KubeClusterName string `yaml:"kubeClusterName" json:"kubeClusterName,omitempty"` Network NetworkConfig `yaml:"network" json:"network,omitempty"` } -type HostCfg struct { - HostName string `yaml:"hostName,omitempty" json:"hostName,omitempty"` - Address string `yaml:"address" json:"address,omitempty"` - Port string `yaml:"port" json:"port,omitempty"` - InternalAddress string `yaml:"internal_address" json:"internalAddress,omitempty"` - Role []string `yaml:"role" json:"role,omitempty" norman:"type=array[enum],options=etcd|worker|worker"` - //HostnameOverride string `yaml:"hostname_override" json:"hostnameOverride,omitempty"` - User string `yaml:"user" json:"user,omitempty"` - Password string `yaml:"password" json:"password,omitempty"` - //SSHAgentAuth bool `yaml:"ssh_agent_auth,omitempty" json:"sshAgentAuth,omitempty"` - //SSHKey string `yaml:"ssh_key" json:"sshKey,omitempty" norman:"type=password"` - SSHKeyPath string `yaml:"ssh_key_path" json:"sshKeyPath,omitempty"` - //SSHCert string `yaml:"ssh_cert" json:"sshCert,omitempty"` - //SSHCertPath string `yaml:"ssh_cert_path" json:"sshCertPath,omitempty"` - //Labels map[string]string `yaml:"labels" json:"labels,omitempty"` - //Taints []Taint `yaml:"taints" json:"taints,omitempty"` - ID int `json:"-"` +func (c ClusterCfg) GetObjectKind() schema.ObjectKind { + panic("implement me") +} + +func (c ClusterCfg) DeepCopyObject() runtime.Object { + panic("implement me") } type Taint struct { @@ -57,3 +76,33 @@ type LBKubeApiserverCfg struct { Address string `yaml:"address" json:"address,omitempty"` Port string `yaml:"port" json:"port,omitempty"` } + +func addDefaultingFuncs(scheme *runtime.Scheme) error { + return RegisterDefaults(scheme) +} + +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&ClusterCfg{}, func(obj interface{}) { SetDefaultClusterCfg(obj.(*ClusterCfg)) }) + return nil +} + +func (cfg *ClusterCfg) GenerateHosts() []string { + var lbHost string + hostsList := []string{} + + _, _, masters, _, _ := cfg.GroupHosts() + if cfg.LBKubeApiserver.Address != "" { + lbHost = fmt.Sprintf("%s %s", cfg.LBKubeApiserver.Address, cfg.LBKubeApiserver.Domain) + } else { + lbHost = fmt.Sprintf("%s %s", masters.Hosts[0].InternalAddress, DefaultLBDomain) + } + + for _, host := range cfg.Hosts { + if host.HostName != "" { + hostsList = append(hostsList, fmt.Sprintf("%s %s.%s %s", host.InternalAddress, host.HostName, cfg.KubeClusterName, host.HostName)) + } + } + + hostsList = append(hostsList, lbHost) + return hostsList +} diff --git a/apis/v1alpha1/config.go b/apis/v1alpha1/config.go index 0be26204..15f6e434 100644 --- a/apis/v1alpha1/config.go +++ b/apis/v1alpha1/config.go @@ -1,66 +1,155 @@ package v1alpha1 import ( - "fmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "io/ioutil" - "os" - "path/filepath" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "strconv" ) -func GetClusterCfg(clusterCfgFile string) *ClusterCfg { - if clusterCfgFile != "" { - clusterInfo, err := ResolveClusterInfoFile(clusterCfgFile) - if err != nil { - log.Fatal("Failed to parse the configuration file: ", err) - } - return clusterInfo - } else { - clusterInfo := &ClusterCfg{ - Hosts: []HostCfg{{ - Role: []string{"etcd", "master", "worker"}, - User: "root", - }}, - Network: NetworkConfig{ - Plugin: DefaultNetworkPlugin, - KubePodsCIDR: DefaultPodsCIDR, - KubeServiceCIDR: DefaultServiceCIDR, - }, - } - return clusterInfo +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme, serializer.EnableStrict) + +func LoadClusterCfg(clusterCfgPath string, logger *log.Logger) (*ClusterCfg, error) { + if len(clusterCfgPath) == 0 { + return nil, errors.New("cluster configuration path not provided") } + + cluster, err := ioutil.ReadFile(clusterCfgPath) + if err != nil { + return nil, errors.Wrap(err, "unable to read the given cluster configuration file") + } + + return ParseClusterCfg(cluster) } -func ResolveClusterInfoFile(configFile string) (*ClusterCfg, error) { - fp, err := filepath.Abs(configFile) - if err != nil { - return nil, fmt.Errorf("failed to lookup current directory name: %v", err) - } - file, err := os.Open(fp) - if err != nil { - return nil, fmt.Errorf("can not find cluster info file: %v", err) - } - defer file.Close() - - clusterInfo, err := GetYamlFile(configFile) - if err != nil { - return nil, fmt.Errorf("failed to read file: %v", err) - } - - return clusterInfo, nil -} - -func GetYamlFile(filePath string) (*ClusterCfg, error) { - result := ClusterCfg{} - b, err := ioutil.ReadFile(filePath) - if err != nil { +func ParseClusterCfg(cluster []byte) (*ClusterCfg, error) { + initCfg := &ClusterCfg{} + if err := runtime.DecodeInto(Codecs.UniversalDecoder(), cluster, initCfg); err != nil { return nil, err } - //var m HostJson - err = yaml.Unmarshal(b, &result) - if err != nil { - return nil, err - } - return &result, nil + + return GetDefaultClusterCfg(initCfg) +} + +func GetDefaultClusterCfg(cfg *ClusterCfg) (*ClusterCfg, error) { + internalCfg := &ClusterCfg{} + + // Default and convert to the internal API type + Scheme.Default(cfg) + if err := Scheme.Convert(cfg, internalCfg, nil); err != nil { + return nil, errors.Wrap(err, "unable to convert versioned to internal cluster object") + } + + return internalCfg, nil +} + +func SetDefaultClusterCfg(cfg *ClusterCfg) *ClusterCfg { + clusterCfg := &ClusterCfg{} + + cfg.Hosts = SetDefaultHostsCfg(cfg) + cfg.LBKubeApiserver = SetDefaultLBCfg(cfg) + cfg.Network = SetDefaultNetworkCfg(cfg) + + if cfg.KubeImageRepo == "" { + cfg.KubeImageRepo = DefaultKubeImageRepo + } + if cfg.KubeClusterName == "" { + cfg.KubeClusterName = DefaultClusterName + } + if cfg.KubeVersion == "" { + cfg.KubeVersion = DefaultKubeVersion + } + clusterCfg = cfg + return clusterCfg +} + +func SetDefaultHostsCfg(cfg *ClusterCfg) []HostCfg { + var hostscfg []HostCfg + if len(cfg.Hosts) == 0 { + return nil + } + for index, host := range cfg.Hosts { + host.ID = index + + if len(host.SSHAddress) == 0 && len(host.InternalAddress) > 0 { + host.SSHAddress = host.InternalAddress + } + if len(host.InternalAddress) == 0 && len(host.SSHAddress) > 0 { + host.InternalAddress = host.SSHAddress + } + if host.User == "" { + host.User = "root" + } + if host.Port == "" { + host.Port = strconv.Itoa(22) + } + + for _, role := range host.Role { + if role == "etcd" { + host.IsEtcd = true + } + if role == "master" { + host.IsMaster = true + } + if role == "worker" { + host.IsWorker = true + } + } + + hostscfg = append(hostscfg, host) + } + + return hostscfg +} + +func SetDefaultLBCfg(cfg *ClusterCfg) LBKubeApiserverCfg { + masterHosts := []HostCfg{} + hosts := SetDefaultHostsCfg(cfg) + for _, host := range hosts { + for _, role := range host.Role { + if role == "etcd" { + host.IsEtcd = true + } + if role == "master" { + host.IsMaster = true + } + if role == "worker" { + host.IsWorker = true + } + } + if host.IsMaster { + masterHosts = append(masterHosts, host) + } + } + + if cfg.LBKubeApiserver.Address == "" { + cfg.LBKubeApiserver.Address = masterHosts[0].InternalAddress + } + if cfg.LBKubeApiserver.Domain == "" { + cfg.LBKubeApiserver.Domain = DefaultLBDomain + } + if cfg.LBKubeApiserver.Port == "" { + cfg.LBKubeApiserver.Port = DefaultLBPort + } + defaultLbCfg := cfg.LBKubeApiserver + return defaultLbCfg +} + +func SetDefaultNetworkCfg(cfg *ClusterCfg) NetworkConfig { + if cfg.Network.Plugin == "" { + cfg.Network.Plugin = DefaultNetworkPlugin + } + if cfg.Network.KubePodsCIDR == "" { + cfg.Network.KubePodsCIDR = DefaultPodsCIDR + } + if cfg.Network.KubeServiceCIDR == "" { + cfg.Network.KubeServiceCIDR = DefaultServiceCIDR + } + + defaultNetworkCfg := cfg.Network + + return defaultNetworkCfg } diff --git a/apis/v1alpha1/default.go b/apis/v1alpha1/default.go index fb90e439..b657c762 100644 --- a/apis/v1alpha1/default.go +++ b/apis/v1alpha1/default.go @@ -1,44 +1,44 @@ package v1alpha1 -const ( - DefaultPreDir = "/tmp/kubekey" - DefaultSSHPort = "22" - DefaultDockerSockPath = "/var/run/docker.sock" - DefaultLBPort = "6443" - DefaultLBDomain = "lb.kubesphere.local" - DefaultNetworkPlugin = "calico" - DefaultPodsCIDR = "10.233.64.0/18" - DefaultServiceCIDR = "10.233.0.0/18" - DefaultKubeImageRepo = "kubekey" - DefaultClusterName = "cluster.local" - DefaultArch = "amd64" - DefaultHostName = "allinone" - DefaultEtcdRepo = "kubekey/etcd" - DefaultEtcdVersion = "v3.3.12" - DefaultEtcdPort = "2379" - DefaultKubeVersion = "v1.17.4" - DefaultCniVersion = "v0.8.2" - DefaultHelmVersion = "v3.1.2" - ETCDRole = "etcd" - MasterRole = "master" - WorkerRole = "worker" -) - -type HostConfig struct { - ID int `json:"-"` - PublicAddress string `json:"publicAddress"` - PrivateAddress string `json:"privateAddress"` - SSHPort int `json:"sshPort"` - SSHUsername string `json:"sshUsername"` - SSHPrivateKeyFile string `json:"sshPrivateKeyFile"` - SSHAgentSocket string `json:"sshAgentSocket"` - Bastion string `json:"bastion"` - BastionPort int `json:"bastionPort"` - BastionUser string `json:"bastionUser"` - Hostname string `json:"hostname"` - IsLeader bool `json:"isLeader"` - Untaint bool `json:"untaint"` - - // Information populated at the runtime - OperatingSystem string `json:"-"` -} +//const ( +// DefaultPreDir = "/tmp/kubekey" +// DefaultSSHPort = "22" +// DefaultDockerSockPath = "/var/run/docker.sock" +// DefaultLBPort = "6443" +// DefaultLBDomain = "lb.kubesphere.local" +// DefaultNetworkPlugin = "calico" +// DefaultPodsCIDR = "10.233.64.0/18" +// DefaultServiceCIDR = "10.233.0.0/18" +// DefaultKubeImageRepo = "kubekey" +// DefaultClusterName = "cluster.local" +// DefaultArch = "amd64" +// DefaultHostName = "allinone" +// DefaultEtcdRepo = "kubekey/etcd" +// DefaultEtcdVersion = "v3.3.12" +// DefaultEtcdPort = "2379" +// DefaultKubeVersion = "v1.17.4" +// DefaultCniVersion = "v0.8.2" +// DefaultHelmVersion = "v3.1.2" +// ETCDRole = "etcd" +// MasterRole = "master" +// WorkerRole = "worker" +//) +// +//type HostConfig struct { +// ID int `json:"-"` +// PublicAddress string `json:"publicAddress"` +// PrivateAddress string `json:"privateAddress"` +// SSHPort int `json:"sshPort"` +// SSHUsername string `json:"sshUsername"` +// SSHPrivateKeyFile string `json:"sshPrivateKeyFile"` +// SSHAgentSocket string `json:"sshAgentSocket"` +// Bastion string `json:"bastion"` +// BastionPort int `json:"bastionPort"` +// BastionUser string `json:"bastionUser"` +// Hostname string `json:"hostname"` +// IsLeader bool `json:"isLeader"` +// Untaint bool `json:"untaint"` +// +// // Information populated at the runtime +// OperatingSystem string `json:"-"` +//} diff --git a/apis/v1alpha1/host.go b/apis/v1alpha1/host.go new file mode 100644 index 00000000..f727f5df --- /dev/null +++ b/apis/v1alpha1/host.go @@ -0,0 +1,58 @@ +package v1alpha1 + +type HostCfg struct { + HostName string `yaml:"hostName,omitempty" json:"hostName,omitempty"` + SSHAddress string `yaml:"sshAddress" json:"sshAddress,omitempty"` + InternalAddress string `yaml:"internalAddress" json:"internalAddress,omitempty"` + Port string `yaml:"port" json:"port,omitempty"` + User string `yaml:"user" json:"user,omitempty"` + Password string `yaml:"password" json:"password,omitempty"` + SSHKeyPath string `yaml:"sshKeyPath" json:"sshKeyPath,omitempty"` + Role []string `yaml:"role" json:"role,omitempty" norman:"type=array[enum],options=etcd|master|worker"` + ID int `json:"-"` + IsEtcd bool + IsMaster bool + IsWorker bool + OSFamily string +} + +type Hosts struct { + Hosts []HostCfg +} + +func (cfg *ClusterCfg) GroupHosts() (*Hosts, *Hosts, *Hosts, *Hosts, *Hosts) { + allHosts := Hosts{} + etcdHosts := Hosts{} + masterHosts := Hosts{} + workerHosts := Hosts{} + k8sHosts := Hosts{} + + for _, host := range cfg.Hosts { + //clusterNode := HostCfg{} + for _, role := range host.Role { + if role == "etcd" { + host.IsEtcd = true + } + if role == "master" { + host.IsMaster = true + } + if role == "worker" { + host.IsWorker = true + } + } + if host.IsEtcd == true { + etcdHosts.Hosts = append(etcdHosts.Hosts, host) + } + if host.IsMaster == true { + masterHosts.Hosts = append(masterHosts.Hosts, host) + } + if host.IsWorker == true { + workerHosts.Hosts = append(workerHosts.Hosts, host) + } + if host.IsMaster == true || host.IsWorker == true { + k8sHosts.Hosts = append(k8sHosts.Hosts, host) + } + allHosts.Hosts = append(allHosts.Hosts, host) + } + return &allHosts, &etcdHosts, &masterHosts, &workerHosts, &k8sHosts +} diff --git a/cluster/container-engine/docker/docker.go b/cluster/container-engine/docker/docker.go index c2bd63de..fab62faa 100644 --- a/cluster/container-engine/docker/docker.go +++ b/cluster/container-engine/docker/docker.go @@ -2,29 +2,22 @@ package docker import ( kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" - "github.com/pixiake/kubekey/util/dialer/ssh" - "github.com/pixiake/kubekey/util/state" + "github.com/pixiake/kubekey/util/manager" + "github.com/pixiake/kubekey/util/ssh" "github.com/pkg/errors" ) -func InstallerDocker(s *state.State) error { - s.Logger.Infoln("Installing docker……") +func InstallerDocker(mgr *manager.Manager) error { + mgr.Logger.Infoln("Installing docker……") - return s.RunTaskOnAllNodes(installDockerOnNode, true) + return mgr.RunTaskOnAllNodes(installDockerOnNode, true) } -func installDockerOnNode(s *state.State, node *kubekeyapi.HostCfg, conn ssh.Connection) error { - err := installDocker(s) +func installDockerOnNode(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + cmd := "sudo sh -c \"[ -z $(which docker) ] && curl https://raw.githubusercontent.com/pixiake/kubeocean/master/scripts/docker-install.sh | sh ; systemctl enable docker\"" + _, err := mgr.Runner.RunRaw(cmd) if err != nil { - return errors.Wrap(err, "failed to install docker") + return errors.Wrap(errors.WithStack(err), "failed to install docker") } return nil } - -func installDocker(s *state.State) error { - cmd := "sudo sh -c \"[ -z $(which docker) ] && curl https://raw.githubusercontent.com/pixiake/kubeocean/master/scripts/docker-install.sh | sh ; systemctl enable docker\"" - //cmd := "[ -z $(which docker) ] && curl https://raw.githubusercontent.com/pixiake/kubeocean/master/scripts/docker-install.sh | sh ; systemctl enable docker" - _, _, err := s.Runner.RunRaw(cmd) - - return errors.WithStack(err) -} diff --git a/cluster/kubernetes/master.go b/cluster/kubernetes/master.go new file mode 100644 index 00000000..cb00899d --- /dev/null +++ b/cluster/kubernetes/master.go @@ -0,0 +1,5 @@ +package kubernetes + +func SyncKubeFiles() { + +} diff --git a/cluster/kubernetes/nodes.go b/cluster/kubernetes/nodes.go new file mode 100644 index 00000000..3bb76fb6 --- /dev/null +++ b/cluster/kubernetes/nodes.go @@ -0,0 +1,94 @@ +package kubernetes + +import ( + "encoding/base64" + "fmt" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/cluster/kubernetes/tmpl" + "github.com/pixiake/kubekey/util/manager" + "github.com/pixiake/kubekey/util/ssh" + "github.com/pkg/errors" + "os" + "path/filepath" + "strings" +) + +func SyncKubeBinaries(mgr *manager.Manager) error { + mgr.Logger.Infoln("Syncing kube binaries……") + + return mgr.RunTaskOnAllNodes(syncKubeBinaries, true) +} + +func syncKubeBinaries(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + currentDir, err1 := filepath.Abs(filepath.Dir(os.Args[0])) + if err1 != nil { + return errors.Wrap(err1, "faild get current dir") + } + + filepath := fmt.Sprintf("%s/%s", currentDir, kubekeyapi.DefaultPreDir) + + kubeadm := fmt.Sprintf("kubeadm-%s", mgr.Cluster.KubeVersion) + kubelet := fmt.Sprintf("kubelet-%s", mgr.Cluster.KubeVersion) + kubectl := fmt.Sprintf("kubectl-%s", mgr.Cluster.KubeVersion) + helm := fmt.Sprintf("helm-%s", kubekeyapi.DefaultHelmVersion) + kubecni := fmt.Sprintf("cni-plugins-linux-%s-%s.tgz", kubekeyapi.DefaultArch, kubekeyapi.DefaultCniVersion) + binaryList := []string{kubeadm, kubelet, kubectl, helm, kubecni} + + for _, binary := range binaryList { + err2 := mgr.Runner.ScpFile(fmt.Sprintf("%s/%s", filepath, binary), fmt.Sprintf("%s/%s", "/tmp/kubekey", binary)) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), fmt.Sprintf("failed to sync binarys")) + } + } + + cmdlist := []string{} + + for _, binary := range binaryList { + if strings.Contains(binary, "cni-plugins-linux") { + cmdlist = append(cmdlist, fmt.Sprintf("mkdir -p /opt/cni/bin && tar -zxf %s/%s -C /opt/cni/bin", "/tmp/kubekey", binary)) + } else { + cmdlist = append(cmdlist, fmt.Sprintf("cp /tmp/kubekey/%s /usr/local/bin/%s && chmod +x /usr/local/bin/%s", binary, strings.Split(binary, "-")[0], strings.Split(binary, "-")[0])) + } + } + cmd := strings.Join(cmdlist, " && ") + _, err3 := mgr.Runner.RunRaw(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", cmd)) + if err3 != nil { + return errors.Wrap(errors.WithStack(err3), fmt.Sprintf("failed to create kubelet link")) + } + return nil +} + +func ConfigureKubeletService(mgr *manager.Manager) error { + mgr.Logger.Infoln("Configure kubelet service……") + + return mgr.RunTaskOnAllNodes(setKubelet, true) +} + +func setKubelet(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + kubeletService, err1 := tmpl.GenerateKubeletService(mgr.Cluster) + if err1 != nil { + return err1 + } + kubeletServiceBase64 := base64.StdEncoding.EncodeToString([]byte(kubeletService)) + _, err2 := mgr.Runner.RunRaw(fmt.Sprintf("echo %s | base64 -d > %s/kubelet.service", kubeletServiceBase64, "/tmp/kubekey")) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to generate kubelet service") + } + + kubeletEnv, err3 := tmpl.GenerateKubeletEnv(mgr.Cluster) + if err3 != nil { + return err3 + } + kubeletEnvBase64 := base64.StdEncoding.EncodeToString([]byte(kubeletEnv)) + _, err4 := mgr.Runner.RunRaw(fmt.Sprintf("echo %s | base64 -d > %s/10-kubeadm.conf", kubeletEnvBase64, "/tmp/kubekey")) + if err4 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to generate kubelet env") + } + + _, err5 := mgr.Runner.RunRaw("sudo -E /bin/sh -c \"cp -f /tmp/kubekey/kubelet.service /etc/systemd/system && mkdir -p /etc/systemd/system/kubelet.service.d && cp -f /tmp/kubekey/10-kubeadm.conf /etc/systemd/system/kubelet.service.d\"") + if err5 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to configure kubelet service") + } + + return nil +} diff --git a/cluster/kubernetes/tmpl/kubeadm.go b/cluster/kubernetes/tmpl/kubeadm.go new file mode 100644 index 00000000..e693b3d1 --- /dev/null +++ b/cluster/kubernetes/tmpl/kubeadm.go @@ -0,0 +1 @@ +package tmpl diff --git a/cluster/kubernetes/tmpl/kubelet.go b/cluster/kubernetes/tmpl/kubelet.go new file mode 100644 index 00000000..609c1faa --- /dev/null +++ b/cluster/kubernetes/tmpl/kubelet.go @@ -0,0 +1,47 @@ +package tmpl + +import ( + "github.com/lithammer/dedent" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/util" + "text/template" +) + +var ( + KubeletServiceTempl = template.Must(template.New("kubeletService").Parse( + dedent.Dedent(`[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ + +[Service] +ExecStart=/usr/local/bin/kubelet +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target + `))) + + KubeletEnvTempl = template.Must(template.New("kubeletEnv").Parse( + dedent.Dedent(`# Note: This dropin only works with kubeadm and kubelet v1.11+ +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS + `))) +) + +func GenerateKubeletService(cfg *kubekeyapi.ClusterCfg) (string, error) { + return util.Render(KubeletServiceTempl, util.Data{}) +} + +func GenerateKubeletEnv(cfg *kubekeyapi.ClusterCfg) (string, error) { + return util.Render(KubeletEnvTempl, util.Data{}) +} diff --git a/cluster/preinstall/initOS.go b/cluster/preinstall/initOS.go new file mode 100644 index 00000000..aba7162b --- /dev/null +++ b/cluster/preinstall/initOS.go @@ -0,0 +1,42 @@ +package preinstall + +import ( + "encoding/base64" + "fmt" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/cluster/preinstall/tmpl" + "github.com/pixiake/kubekey/util/manager" + "github.com/pixiake/kubekey/util/ssh" + "github.com/pkg/errors" +) + +func InitOS(mgr *manager.Manager) error { + mgr.Logger.Infoln("Initialize operating system") + + return mgr.RunTaskOnAllNodes(initOsOnNode, false) +} + +func initOsOnNode(mgr *manager.Manager, node *kubekeyapi.HostCfg, conn ssh.Connection) error { + tmpDir := "/tmp/kubekey" + _, err := mgr.Runner.RunRaw(fmt.Sprintf("mkdir -p %s", tmpDir)) + if err != nil { + return errors.Wrap(errors.WithStack(err), "failed to init operating system") + } + + initOsScript, err1 := tmpl.InitOsScript(mgr.Cluster) + if err1 != nil { + return err1 + } + + str := base64.StdEncoding.EncodeToString([]byte(initOsScript)) + _, err2 := mgr.Runner.RunRaw(fmt.Sprintf("echo %s | base64 -d > %s/initOS.sh && chmod +x %s/initOS.sh", str, tmpDir, tmpDir)) + if err2 != nil { + return errors.Wrap(errors.WithStack(err2), "failed to init operating system") + } + + _, err3 := mgr.Runner.RunRaw(fmt.Sprintf("sudo %s/initOS.sh", tmpDir)) + if err3 != nil { + return errors.Wrap(errors.WithStack(err3), "failed to init operating system") + } + return nil +} diff --git a/cluster/preinstall/preinstall.go b/cluster/preinstall/preinstall.go new file mode 100644 index 00000000..74d6d5ac --- /dev/null +++ b/cluster/preinstall/preinstall.go @@ -0,0 +1,95 @@ +package preinstall + +import ( + "fmt" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/util" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "os" + "os/exec" + "path/filepath" +) + +func FilesDownloadHttp(cfg *kubekeyapi.ClusterCfg, filepath string, logger *log.Logger) error { + kubeVersion := cfg.KubeVersion + + kubeadmUrl := fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubeadm", kubeVersion, kubekeyapi.DefaultArch) + kubeletUrl := fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubelet", kubeVersion, kubekeyapi.DefaultArch) + kubectlUrl := fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubectl", kubeVersion, kubekeyapi.DefaultArch) + kubeCniUrl := fmt.Sprintf("https://containernetworking.pek3b.qingstor.com/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", kubekeyapi.DefaultCniVersion, kubekeyapi.DefaultArch, kubekeyapi.DefaultCniVersion) + HelmUrl := fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-amd64/%s/helm", kubekeyapi.DefaultHelmVersion) + + kubeadm := fmt.Sprintf("%s/kubeadm-%s", filepath, kubeVersion) + kubelet := fmt.Sprintf("%s/kubelet-%s", filepath, kubeVersion) + kubectl := fmt.Sprintf("%s/kubectl-%s", filepath, kubeVersion) + kubeCni := fmt.Sprintf("%s/cni-plugins-linux-%s-%s.tgz", filepath, kubekeyapi.DefaultArch, kubekeyapi.DefaultCniVersion) + helm := fmt.Sprintf("%s/helm-%s", filepath, kubekeyapi.DefaultHelmVersion) + + getKubeadmCmd := fmt.Sprintf("curl -o %s %s", kubeadm, kubeadmUrl) + getKubeletCmd := fmt.Sprintf("curl -o %s %s", kubelet, kubeletUrl) + getKubectlCmd := fmt.Sprintf("curl -o %s %s", kubectl, kubectlUrl) + getKubeCniCmd := fmt.Sprintf("curl -o %s %s", kubeCni, kubeCniUrl) + getHelmCmd := fmt.Sprintf("curl -o %s %s", helm, HelmUrl) + + logger.Info("Kubeadm being download ...") + if util.IsExist(kubeadm) == false { + if out, err := exec.Command("/bin/sh", "-c", getKubeadmCmd).CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "faild download kubeadm binary") + } + } + + logger.Info("Kubelet being download ...") + if util.IsExist(kubelet) == false { + if out, err := exec.Command("/bin/sh", "-c", getKubeletCmd).CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "faild download kubelet binary") + } + } + + logger.Info("Kubectl being download ...") + if util.IsExist(kubectl) == false { + if out, err := exec.Command("/bin/sh", "-c", getKubectlCmd).CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "faild download kubectl binary") + } + } + + logger.Info("KubeCni being download ...") + if util.IsExist(kubeCni) == false { + if out, err := exec.Command("/bin/sh", "-c", getKubeCniCmd).CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "faild download kubecni") + } + } + + logger.Info("Helm being download ...") + if util.IsExist(helm) == false { + if out, err := exec.Command("/bin/sh", "-c", getHelmCmd).CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "faild download helm binary") + } + } + + return nil +} + +func Prepare(cfg *kubekeyapi.ClusterCfg, logger *log.Logger) error { + logger.Info("Install Files Download") + + currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return errors.Wrap(err, "faild get current dir") + } + + filepath := fmt.Sprintf("%s/%s", currentDir, kubekeyapi.DefaultPreDir) + if err := util.CreateDir(filepath); err != nil { + return errors.Wrap(err, "faild create download target dir") + } + + if err := FilesDownloadHttp(cfg, filepath, logger); err != nil { + return err + } + return nil +} diff --git a/cluster/preinstall/tmpl/InitOS.go b/cluster/preinstall/tmpl/InitOS.go new file mode 100644 index 00000000..4496368c --- /dev/null +++ b/cluster/preinstall/tmpl/InitOS.go @@ -0,0 +1,80 @@ +package tmpl + +import ( + "github.com/lithammer/dedent" + kubekeyapi "github.com/pixiake/kubekey/apis/v1alpha1" + "github.com/pixiake/kubekey/util" + "text/template" +) + +var initOsScriptTmpl = template.Must(template.New("initOS").Parse( + dedent.Dedent(`#!/bin/sh + +swapoff -a +sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab + +echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf +echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf +echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf + +sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?(0|1)@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf + +awk ' !x[$0]++{print > "/etc/sysctl.conf"}' /etc/sysctl.conf +sysctl -p + +systemctl stop firewald 1>/dev/null 2>/dev/null +systemctl disable firewald 1>/dev/null 2>/dev/null +systemctl stop ufw 1>/dev/null 2>/dev/null +systemctl disable ufw 1>/dev/null 2>/dev/null + +modinfo br_netfilter > /dev/null 2>&1 +if [ $? -eq 0 ]; then + modprobe br_netfilter + mkdir -p /etc/modules-load.d + echo 'br_netfilter' > /etc/modules-load.d/kubeocean-br_netfilter.conf +fi + +modprobe ip_vs +modprobe ip_vs_rr +modprobe ip_vs_wrr +modprobe ip_vs_sh + +cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF +ip_vs +ip_vs_rr +ip_vs_wrr +ip_vs_sh +EOF + +modprobe nf_conntrack_ipv4 +if [ $? -eq 0 ]; then + echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf +else + modprobe nf_conntrack + echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf +fi + +sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' /etc/hosts +sed -i '/^$/N;/\n$/N;//D' /etc/hosts + +cat >>/etc/hosts<