From 3e5ffdc07da1a53eb8e8fac66710b80d3f3ee74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CForest-L?= Date: Mon, 28 Sep 2020 00:19:14 +0800 Subject: [PATCH] update etcd backup parameter exposure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: “Forest-L --- api/v1alpha1/cluster_types.go | 1 + api/v1alpha1/default.go | 22 ++++++++++++++++ api/v1alpha1/network_types.go | 7 +++++ pkg/cluster/etcd/etcd.go | 12 ++++----- pkg/cluster/etcd/tmpl/etcdbackup.go | 36 +++++++++++++++++++++----- pkg/cluster/preinstall/tmpl/init_os.go | 1 + 6 files changed, 67 insertions(+), 12 deletions(-) diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index 7ed67e1a..db043aba 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -43,6 +43,7 @@ type ClusterSpec struct { Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` Addons []Addon `yaml:"addons" json:"addons,omitempty"` KubeSphere KubeSphere `json:"kubesphere,omitempty"` + Etcd EtcdCfg `yaml:"etcd" json:"etcd,omitempty"` } // ClusterStatus defines the observed state of Cluster diff --git a/api/v1alpha1/default.go b/api/v1alpha1/default.go index 6cd50e56..40f8c6db 100644 --- a/api/v1alpha1/default.go +++ b/api/v1alpha1/default.go @@ -50,6 +50,10 @@ const ( Master = "master" Worker = "worker" K8s = "k8s" + DefaultEtcdBackupDir = "/var/backups/kube_etcd" + DefaultEtcdBackupPeriod = "1" + DefaultKeepBackNumber = "5" + DefaultEtcdBackupScript = "/usr/local/bin/kube-scripts" ) func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, *HostGroups) { @@ -61,6 +65,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, *HostGroups) { clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, hostGroups.Master) clusterCfg.Network = SetDefaultNetworkCfg(cfg) + clusterCfg.Etcd = SetDefaultEtcdCfg(cfg) clusterCfg.Kubernetes = SetDefaultClusterCfg(cfg) clusterCfg.Registry = cfg.Registry clusterCfg.Addons = cfg.Addons @@ -172,3 +177,20 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes { return defaultClusterCfg } + +func SetDefaultEtcdCfg(cfg *ClusterSpec) EtcdCfg { + if cfg.Etcd.EtcdBackupDir == "" { + cfg.Etcd.EtcdBackupDir = DefaultEtcdBackupDir + } + if cfg.Etcd.EtcdBackupPeriod == "" { + cfg.Etcd.EtcdBackupPeriod = DefaultEtcdBackupPeriod + } + if cfg.Etcd.KeepBackupNumber == "" { + cfg.Etcd.KeepBackupNumber = DefaultKeepBackNumber + } + if cfg.Etcd.EtcdBackupScript == "" { + cfg.Etcd.EtcdBackupScript = DefaultEtcdBackupScript + } + defaultEtcdCfg := cfg.Etcd + return defaultEtcdCfg +} diff --git a/api/v1alpha1/network_types.go b/api/v1alpha1/network_types.go index 111fdc81..1def294a 100644 --- a/api/v1alpha1/network_types.go +++ b/api/v1alpha1/network_types.go @@ -28,3 +28,10 @@ type CalicoCfg struct { VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` VethMTU string `yaml:"vethMTU" json:"vethMTU,omitempty"` } + +type EtcdCfg struct { + EtcdBackupDir string `yaml:"etcdBackupDir" json:"etcdBackupDir,omitempty"` + EtcdBackupPeriod string `yaml:"etcdBackupPeriod" json:"etcdBackupPeriod,omitempty"` + KeepBackupNumber string `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"` + EtcdBackupScript string `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"` +} diff --git a/pkg/cluster/etcd/etcd.go b/pkg/cluster/etcd/etcd.go index 6ff02f04..2465ee62 100644 --- a/pkg/cluster/etcd/etcd.go +++ b/pkg/cluster/etcd/etcd.go @@ -267,21 +267,21 @@ func BackupEtcd(mgr *manager.Manager) error { } func backupEtcd(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error { - _, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"if [ -d /opt/etcd_back ]; then rm -rf /opt/etcd_back ;fi && mkdir -p /opt/etcd_back\"", 0, false) + _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"mkdir -p %s\"", mgr.Cluster.Etcd.EtcdBackupScript), 0, false) if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to mkdir /opt/etcd_back") + return errors.Wrap(errors.WithStack(err), "Failed to create etcd backup") } + tmpDir := "/tmp/kubekey" etcdBackupScript, _ := tmpl.EtcdBackupScript(mgr, node) etcdBackupScriptBase64 := base64.StdEncoding.EncodeToString([]byte(etcdBackupScript)) - _, err2 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > /opt/etcd_back/etcd-backup.sh && chmod +x /opt/etcd_back/etcd-backup.sh\"", etcdBackupScriptBase64), 1, false) + _, err2 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > %s/etcd-backup.sh && chmod +x %s/etcd-backup.sh\"", etcdBackupScriptBase64, tmpDir, tmpDir), 1, false) if err2 != nil { return errors.Wrap(errors.WithStack(err2), "Failed to generate etcd backup") } - _, err3 := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"crontab -l | grep -v '#' > /tmp/file;echo '0 2 * * * sh /opt/etcd_back/etcd-backup.sh' >> /tmp/file && awk ' !x[$0]++{print > \"/tmp/file\"}' /tmp/file;crontab /tmp/file\"", 2, false) + _, err3 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo cp %s/etcd-backup.sh %s &&sudo %s/etcd-backup.sh", tmpDir, mgr.Cluster.Etcd.EtcdBackupScript, mgr.Cluster.Etcd.EtcdBackupScript), 1, false) if err3 != nil { - return errors.Wrap(errors.WithStack(err3), "Failed to crontab backup etcd data") + return errors.Wrap(errors.WithStack(err3), "Failed to run the etcd-backup.sh") } - return nil } diff --git a/pkg/cluster/etcd/tmpl/etcdbackup.go b/pkg/cluster/etcd/tmpl/etcdbackup.go index b25fd0f3..f7901405 100644 --- a/pkg/cluster/etcd/tmpl/etcdbackup.go +++ b/pkg/cluster/etcd/tmpl/etcdbackup.go @@ -16,16 +16,30 @@ var EtcdBackupScriptTmpl = template.Must(template.New("etcdBackupScript").Parse( ETCDCTL_PATH='/usr/local/bin/etcdctl' ENDPOINTS='{{ .Etcdendpoint }}' ETCD_DATA_DIR="/var/lib/etcd" -BACKUP_DIR="/var/backups/kube_etcd/etcd-$(date +%Y-%m-%d_%H:%M:%S)" +BACKUP_DIR="{{ .Backupdir }}/etcd-$(date +%Y-%m-%d_%H:%M:%S)" +KEEPBACKUPNUMBER='{{ .KeepbackupNumber }}' +ETCDBACKUPPERIOD='{{ .EtcdBackupPeriod }}' +ETCDBACKUPSCIPT='{{ .EtcdBackupScript }}' + +time="0 2 * * *" +if [[ 1 == $ETCDBACKUPPERIOD ]]; then + time="0 2 * * *" +elif [[ 7 == $ETCDBACKUPPERIOD ]]; then + time="0 2 * * 0" +elif [[ 15 == $ETCDBACKUPPERIOD ]]; then + time="0 2 * * 0" +elif [[ 30 == $ETCDBACKUPPERIOD ]]; then + time="0 2 1 * *" +else + time="0 2 * * *" +fi ETCDCTL_CERT="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem" ETCDCTL_KEY="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem" ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.pem" - [ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR - export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR sleep 3 @@ -39,7 +53,13 @@ export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKU sleep 3 -cd $BACKUP_DIR/../;ls -lt |awk '{if(NR>14){print "rm -rf "$9}}'|sh +cd $BACKUP_DIR/../;ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh + +crontab -l | grep -v '#' > /tmp/file +echo "$time sh $ETCDBACKUPSCIPT/etcd-backup.sh" >> /tmp/file && awk ' !x[$0]++{print > "/tmp/file"}' /tmp/file +crontab /tmp/file +rm -rf /tmp/file + `))) func EtcdBackupScript(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) (string, error) { @@ -48,7 +68,11 @@ func EtcdBackupScript(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) (s ips = append(ips, fmt.Sprintf("https://%s:2379", host.InternalAddress)) } return util.Render(EtcdBackupScriptTmpl, util.Data{ - "Hostname": node.Name, - "Etcdendpoint": strings.Join(ips, ","), + "Hostname": node.Name, + "Etcdendpoint": strings.Join(ips, ","), + "Backupdir": mgr.Cluster.Etcd.EtcdBackupDir, + "KeepbackupNumber": mgr.Cluster.Etcd.KeepBackupNumber, + "EtcdBackupPeriod": mgr.Cluster.Etcd.EtcdBackupPeriod, + "EtcdBackupScript": mgr.Cluster.Etcd.EtcdBackupScript, }) } diff --git a/pkg/cluster/preinstall/tmpl/init_os.go b/pkg/cluster/preinstall/tmpl/init_os.go index 18a831f3..a366fcad 100644 --- a/pkg/cluster/preinstall/tmpl/init_os.go +++ b/pkg/cluster/preinstall/tmpl/init_os.go @@ -115,6 +115,7 @@ ulimit -n 65535 crontab -l | grep -v '#' > /tmp/file1 echo "0 3 * * * ps -A -ostat,ppid | grep -e '^[Zz]' | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" >> /tmp/file1 && awk ' !x[$0]++{print > "/tmp/file1"}' /tmp/file1 crontab /tmp/file1 +rm -rf /tmp/file1 `))) func InitOsScript(mgr *manager.Manager) (string, error) {