From 8f5da2394f6a63961793a0f7f2697a90dc652d1b Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Wed, 15 Jan 2025 18:36:12 +0800 Subject: [PATCH] update 12.0 k8s single node & S3 data migration --- manual/repo/docker/cluster/env | 4 + manual/repo/docker/cluster/seafile-server.yml | 2 + manual/repo/k8s/ce/seafile-deployment.yaml | 108 ++++++++ manual/repo/k8s/ce/seafile-env.yaml | 25 ++ .../repo/k8s/ce/seafile-persistentvolume.yaml | 11 + .../k8s/ce/seafile-persistentvolumeclaim.yaml | 10 + manual/repo/k8s/ce/seafile-secret.yaml | 18 ++ manual/repo/k8s/ce/seafile-service.yaml | 13 + .../cluster/seafile-backend-deployment.yaml | 10 + manual/repo/k8s/cluster/seafile-env.yaml | 3 + manual/repo/k8s/cluster/seafile-secret.yaml | 3 + manual/repo/k8s/pro/seafile-deployment.yaml | 158 +++++++++++ manual/repo/k8s/pro/seafile-env.yaml | 36 +++ .../k8s/pro/seafile-persistentvolume.yaml | 11 + .../pro/seafile-persistentvolumeclaim.yaml | 10 + manual/repo/k8s/pro/seafile-secret.yaml | 21 ++ manual/repo/k8s/pro/seafile-service.yaml | 13 + manual/setup/cluster_deploy_with_k8s.md | 14 +- manual/setup/k8s_single_node.md | 136 +++++++++ manual/setup/migrate_backends_data.md | 262 ++++++++---------- mkdocs.yml | 1 + 21 files changed, 710 insertions(+), 159 deletions(-) create mode 100644 manual/repo/k8s/ce/seafile-deployment.yaml create mode 100644 manual/repo/k8s/ce/seafile-env.yaml create mode 100644 manual/repo/k8s/ce/seafile-persistentvolume.yaml create mode 100644 manual/repo/k8s/ce/seafile-persistentvolumeclaim.yaml create mode 100644 manual/repo/k8s/ce/seafile-secret.yaml create mode 100644 manual/repo/k8s/ce/seafile-service.yaml create mode 100644 manual/repo/k8s/pro/seafile-deployment.yaml create mode 100644 manual/repo/k8s/pro/seafile-env.yaml create mode 100644 manual/repo/k8s/pro/seafile-persistentvolume.yaml create mode 100644 manual/repo/k8s/pro/seafile-persistentvolumeclaim.yaml create mode 100644 manual/repo/k8s/pro/seafile-secret.yaml create mode 100644 manual/repo/k8s/pro/seafile-service.yaml create mode 100644 manual/setup/k8s_single_node.md diff --git a/manual/repo/docker/cluster/env b/manual/repo/docker/cluster/env index fd890a7a..7b828476 100644 --- a/manual/repo/docker/cluster/env +++ b/manual/repo/docker/cluster/env @@ -19,6 +19,10 @@ CLUSTER_INIT_ES_PORT=9200 # only valid in config init CLUSTER_MODE=frontend # backend for backend node +# Seafile admin +INIT_SEAFILE_ADMIN_EMAIL=me@example.com +INIT_SEAFILE_ADMIN_PASSWORD=asecret + # Time zone TIME_ZONE=UTC diff --git a/manual/repo/docker/cluster/seafile-server.yml b/manual/repo/docker/cluster/seafile-server.yml index a55ded30..e7b9c940 100644 --- a/manual/repo/docker/cluster/seafile-server.yml +++ b/manual/repo/docker/cluster/seafile-server.yml @@ -22,6 +22,8 @@ services: - CLUSTER_INIT_ES_PORT=${CLUSTER_INIT_ES_PORT:-9200} - CLUSTER_MODE=${CLUSTER_MODE:-frontend} - TIME_ZONE=${TIME_ZONE:-UTC} + - INIT_SEAFILE_ADMIN_EMAIL=${INIT_SEAFILE_ADMIN_EMAIL:-me@example.com} + - INIT_SEAFILE_ADMIN_PASSWORD=${INIT_SEAFILE_ADMIN_PASSWORD:-asecret} - INIT_S3_STORAGE_BACKEND_CONFIG=${INIT_S3_STORAGE_BACKEND_CONFIG:-false} - INIT_S3_COMMIT_BUCKET=${INIT_S3_COMMIT_BUCKET:-} - INIT_S3_FS_BUCKET=${INIT_S3_FS_BUCKET:-} diff --git a/manual/repo/k8s/ce/seafile-deployment.yaml b/manual/repo/k8s/ce/seafile-deployment.yaml new file mode 100644 index 00000000..85a1de79 --- /dev/null +++ b/manual/repo/k8s/ce/seafile-deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seafile +spec: + replicas: 1 + selector: + matchLabels: + app: seafile + template: + metadata: + labels: + app: seafile + spec: + containers: + - name: seafile + image: seafileltd/seafile-pro-mc:12.0-latest + env: + - name: TIME_ZONE + valueFrom: + configMapKeyRef: + name: seafile-env + key: TIME_ZONE + - name: SEAFILE_LOG_TO_STDOUT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_LOG_TO_STDOUT + - name: SITE_ROOT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SITE_ROOT + - name: ENABLE_SEADOC + valueFrom: + configMapKeyRef: + name: seafile-env + key: ENABLE_SEADOC + - name: SEADOC_SERVER_URL + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEADOC_SERVER_URL + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_HOST + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_PORT + - name: DB_USER + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_USER + - name: SEAFILE_MYSQL_DB_CCNET_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_CCNET_DB_NAME + - name: SEAFILE_MYSQL_DB_SEAFILE_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_SEAFILE_DB_NAME + - name: SEAFILE_MYSQL_DB_SEAHUB_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_SEAHUB_DB_NAME + - name: INIT_SEAFILE_ADMIN_EMAIL + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_SEAFILE_ADMIN_EMAIL + - name: JWT_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: seafile-secret + key: JWT_PRIVATE_KEY + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: seafile-secret + key: SEAFILE_MYSQL_DB_PASSWORD + - name: DB_ROOT_PASSWD + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_SEAFILE_MYSQL_ROOT_PASSWORD + - name: INIT_SEAFILE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_SEAFILE_ADMIN_PASSWORD + volumeMounts: + - name: seafile-data + mountPath: /shared + volumes: + - name: seafile-data + persistentVolumeClaim: + claimName: seafile-data + restartPolicy: Always + imagePullSecrets: + - name: regcred \ No newline at end of file diff --git a/manual/repo/k8s/ce/seafile-env.yaml b/manual/repo/k8s/ce/seafile-env.yaml new file mode 100644 index 00000000..8f781560 --- /dev/null +++ b/manual/repo/k8s/ce/seafile-env.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: seafile-env +data: + # for Seafile server + TIME_ZONE: "UTC" + SEAFILE_LOG_TO_STDOUT: "true" + SITE_ROOT: "/" + ENABLE_SEADOC: "false" + SEADOC_SERVER_URL: "https://seafile.example.com/sdoc-server" # only valid in ENABLE_SEADOC = true + SEAFILE_SERVER_HOSTNAME: "seafile.example.com" + + + # for database + SEAFILE_MYSQL_DB_HOST: "" + SEAFILE_MYSQL_DB_PORT: "3306" + SEAFILE_MYSQL_DB_USER: "seafile" + SEAFILE_MYSQL_DB_CCNET_DB_NAME: "ccnet_db" + SEAFILE_MYSQL_DB_SEAFILE_DB_NAME: "seafile_db" + SEAFILE_MYSQL_DB_SEAHUB_DB_NAME: "seahub_db" + + # Init + ## for Seafile admin + INIT_SEAFILE_ADMIN_EMAIL: "" \ No newline at end of file diff --git a/manual/repo/k8s/ce/seafile-persistentvolume.yaml b/manual/repo/k8s/ce/seafile-persistentvolume.yaml new file mode 100644 index 00000000..f1321e8c --- /dev/null +++ b/manual/repo/k8s/ce/seafile-persistentvolume.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: seafile-data +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /opt/seafile-data \ No newline at end of file diff --git a/manual/repo/k8s/ce/seafile-persistentvolumeclaim.yaml b/manual/repo/k8s/ce/seafile-persistentvolumeclaim.yaml new file mode 100644 index 00000000..28d26c02 --- /dev/null +++ b/manual/repo/k8s/ce/seafile-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: seafile-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/manual/repo/k8s/ce/seafile-secret.yaml b/manual/repo/k8s/ce/seafile-secret.yaml new file mode 100644 index 00000000..8e949391 --- /dev/null +++ b/manual/repo/k8s/ce/seafile-secret.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: seafile-secret +type: Opaque +data: + # for Seafile server + JWT_PRIVATE_KEY: "" + + # for database + SEAFILE_MYSQL_DB_PASSWORD: "" + + # Initialization + ## for seafile + INIT_SEAFILE_ADMIN_PASSWORD: "" + + ## for db + INIT_SEAFILE_MYSQL_ROOT_PASSWORD: "" \ No newline at end of file diff --git a/manual/repo/k8s/ce/seafile-service.yaml b/manual/repo/k8s/ce/seafile-service.yaml new file mode 100644 index 00000000..9c5997ba --- /dev/null +++ b/manual/repo/k8s/ce/seafile-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: seafile +spec: + selector: + app: seafile + type: LoadBalancer + ports: + - protocol: TCP + port: 80 + targetPort: 80 + nodePort: 30000 \ No newline at end of file diff --git a/manual/repo/k8s/cluster/seafile-backend-deployment.yaml b/manual/repo/k8s/cluster/seafile-backend-deployment.yaml index ccb6c699..b1fd23a0 100644 --- a/manual/repo/k8s/cluster/seafile-backend-deployment.yaml +++ b/manual/repo/k8s/cluster/seafile-backend-deployment.yaml @@ -147,6 +147,11 @@ spec: configMapKeyRef: name: seafile-env key: INIT_S3_USE_HTTPS + - name: INIT_SEAFILE_ADMIN_EMAIL + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_SEAFILE_ADMIN_EMAIL - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: @@ -167,6 +172,11 @@ spec: secretKeyRef: name: seafile-secret key: INIT_S3_SECRET_KEY + - name: INIT_SEAFILE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_SEAFILE_ADMIN_PASSWORD volumeMounts: - name: seafile-data mountPath: /shared diff --git a/manual/repo/k8s/cluster/seafile-env.yaml b/manual/repo/k8s/cluster/seafile-env.yaml index db6bef74..a2ec0b84 100644 --- a/manual/repo/k8s/cluster/seafile-env.yaml +++ b/manual/repo/k8s/cluster/seafile-env.yaml @@ -20,6 +20,9 @@ data: # initialization (only valid in first-time deployment and CLUSTER_INIT_MODE = true) CLUSTER_INIT_MODE: "true" + + ## for Seafile admin + INIT_SEAFILE_ADMIN_EMAIL: "" ## for cluster basic service CLUSTER_INIT_MEMCACHED_HOST: "" diff --git a/manual/repo/k8s/cluster/seafile-secret.yaml b/manual/repo/k8s/cluster/seafile-secret.yaml index 3dd31485..ef084e92 100644 --- a/manual/repo/k8s/cluster/seafile-secret.yaml +++ b/manual/repo/k8s/cluster/seafile-secret.yaml @@ -11,6 +11,9 @@ data: SEAFILE_MYSQL_DB_PASSWORD: "" # Initialization + ## for Seafile admin + INIT_SEAFILE_ADMIN_PASSWORD: "" + ## for db INIT_SEAFILE_MYSQL_ROOT_PASSWORD: "" diff --git a/manual/repo/k8s/pro/seafile-deployment.yaml b/manual/repo/k8s/pro/seafile-deployment.yaml new file mode 100644 index 00000000..ffd897c4 --- /dev/null +++ b/manual/repo/k8s/pro/seafile-deployment.yaml @@ -0,0 +1,158 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seafile +spec: + replicas: 1 + selector: + matchLabels: + app: seafile + template: + metadata: + labels: + app: seafile + spec: + containers: + - name: seafile + image: seafileltd/seafile-pro-mc:12.0-latest + env: + - name: TIME_ZONE + valueFrom: + configMapKeyRef: + name: seafile-env + key: TIME_ZONE + - name: SEAFILE_LOG_TO_STDOUT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_LOG_TO_STDOUT + - name: SITE_ROOT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SITE_ROOT + - name: ENABLE_SEADOC + valueFrom: + configMapKeyRef: + name: seafile-env + key: ENABLE_SEADOC + - name: SEADOC_SERVER_URL + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEADOC_SERVER_URL + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_HOST + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_PORT + - name: DB_USER + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_USER + - name: SEAFILE_MYSQL_DB_CCNET_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_CCNET_DB_NAME + - name: SEAFILE_MYSQL_DB_SEAFILE_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_SEAFILE_DB_NAME + - name: SEAFILE_MYSQL_DB_SEAHUB_DB_NAME + valueFrom: + configMapKeyRef: + name: seafile-env + key: SEAFILE_MYSQL_DB_SEAHUB_DB_NAME + - name: INIT_S3_STORAGE_BACKEND_CONFIG + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_STORAGE_BACKEND_CONFIG + - name: INIT_S3_COMMIT_BUCKET + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_COMMIT_BUCKET + - name: INIT_S3_FS_BUCKET + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_FS_BUCKET + - name: INIT_S3_BLOCK_BUCKET + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_BLOCK_BUCKET + - name: INIT_S3_KEY_ID + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_KEY_ID + - name: INIT_S3_USE_V4_SIGNATURE + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_USE_V4_SIGNATURE + - name: INIT_S3_AWS_REGION + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_AWS_REGION + - name: INIT_S3_HOST + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_HOST + - name: INIT_S3_USE_HTTPS + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_S3_USE_HTTPS + - name: INIT_SEAFILE_ADMIN_EMAIL + valueFrom: + configMapKeyRef: + name: seafile-env + key: INIT_SEAFILE_ADMIN_EMAIL + - name: JWT_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: seafile-secret + key: JWT_PRIVATE_KEY + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: seafile-secret + key: SEAFILE_MYSQL_DB_PASSWORD + - name: DB_ROOT_PASSWD + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_SEAFILE_MYSQL_ROOT_PASSWORD + - name: INIT_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_S3_SECRET_KEY + - name: INIT_SEAFILE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: seafile-secret + key: INIT_SEAFILE_ADMIN_PASSWORD + volumeMounts: + - name: seafile-data + mountPath: /shared + volumes: + - name: seafile-data + persistentVolumeClaim: + claimName: seafile-data + restartPolicy: Always + imagePullSecrets: + - name: regcred \ No newline at end of file diff --git a/manual/repo/k8s/pro/seafile-env.yaml b/manual/repo/k8s/pro/seafile-env.yaml new file mode 100644 index 00000000..c41e1559 --- /dev/null +++ b/manual/repo/k8s/pro/seafile-env.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: seafile-env +data: + # for Seafile server + TIME_ZONE: "UTC" + SEAFILE_LOG_TO_STDOUT: "true" + SITE_ROOT: "/" + ENABLE_SEADOC: "false" + SEADOC_SERVER_URL: "https://seafile.example.com/sdoc-server" # only valid in ENABLE_SEADOC = true + SEAFILE_SERVER_HOSTNAME: "seafile.example.com" + + + # for database + SEAFILE_MYSQL_DB_HOST: "" + SEAFILE_MYSQL_DB_PORT: "3306" + SEAFILE_MYSQL_DB_USER: "seafile" + SEAFILE_MYSQL_DB_CCNET_DB_NAME: "ccnet_db" + SEAFILE_MYSQL_DB_SEAFILE_DB_NAME: "seafile_db" + SEAFILE_MYSQL_DB_SEAHUB_DB_NAME: "seahub_db" + + # Init + ## for Seafile admin + INIT_SEAFILE_ADMIN_EMAIL: "" + + ## For S3 storage backend (only valid in INIT_S3_STORAGE_BACKEND_CONFIG = true) + INIT_S3_STORAGE_BACKEND_CONFIG: "false" + INIT_S3_COMMIT_BUCKET: "" + INIT_S3_FS_BUCKET: "" + INIT_S3_BLOCK_BUCKET: "" + INIT_S3_KEY_ID: "" + INIT_S3_USE_V4_SIGNATURE: "true" + INIT_S3_AWS_REGION: "us-east-1" + INIT_S3_HOST: "s3.us-east-1.amazonaws.com" + INIT_S3_USE_HTTPS: "true" diff --git a/manual/repo/k8s/pro/seafile-persistentvolume.yaml b/manual/repo/k8s/pro/seafile-persistentvolume.yaml new file mode 100644 index 00000000..f1321e8c --- /dev/null +++ b/manual/repo/k8s/pro/seafile-persistentvolume.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: seafile-data +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /opt/seafile-data \ No newline at end of file diff --git a/manual/repo/k8s/pro/seafile-persistentvolumeclaim.yaml b/manual/repo/k8s/pro/seafile-persistentvolumeclaim.yaml new file mode 100644 index 00000000..28d26c02 --- /dev/null +++ b/manual/repo/k8s/pro/seafile-persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: seafile-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/manual/repo/k8s/pro/seafile-secret.yaml b/manual/repo/k8s/pro/seafile-secret.yaml new file mode 100644 index 00000000..38124c70 --- /dev/null +++ b/manual/repo/k8s/pro/seafile-secret.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Secret +metadata: + name: seafile-secret +type: Opaque +data: + # for Seafile server + JWT_PRIVATE_KEY: "" + + # for database + SEAFILE_MYSQL_DB_PASSWORD: "" + + # Initialization + ## for seafile + INIT_SEAFILE_ADMIN_PASSWORD: "" + + ## for db + INIT_SEAFILE_MYSQL_ROOT_PASSWORD: "" + + ## For S3 storage backend (only valid in INIT_S3_STORAGE_BACKEND_CONFIG = true) + INIT_S3_SECRET_KEY: "" \ No newline at end of file diff --git a/manual/repo/k8s/pro/seafile-service.yaml b/manual/repo/k8s/pro/seafile-service.yaml new file mode 100644 index 00000000..9c5997ba --- /dev/null +++ b/manual/repo/k8s/pro/seafile-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: seafile +spec: + selector: + app: seafile + type: LoadBalancer + ports: + - protocol: TCP + port: 80 + targetPort: 80 + nodePort: 30000 \ No newline at end of file diff --git a/manual/setup/cluster_deploy_with_k8s.md b/manual/setup/cluster_deploy_with_k8s.md index ebf66a88..97fe27f0 100644 --- a/manual/setup/cluster_deploy_with_k8s.md +++ b/manual/setup/cluster_deploy_with_k8s.md @@ -1,6 +1,6 @@ # Deploy Seafile cluster with Kubernetes (K8S) -This manual explains how to deploy and run Seafile Server on a Linux server using *Kubernetes* (***k8s*** thereafter). +This manual explains how to deploy and run Seafile cluster on a Linux server using *Kubernetes* (***k8s*** thereafter). ## Prerequisites @@ -29,7 +29,7 @@ For each node, you have to prepare at least **2 cores** cpu, **2G RAM** and 10G !!! tip "More details about the number of nodes" 1. If your number of nodes does not meet our recommended number (i.e. 3 nodes), please adjust according to the following strategies: - **2 nodes**: A frontend service and a backend service on the same node - - **1 node**: Please refer [here](./setup_pro_by_docker.md) to deploy Seafile in a single node instead a cluster. + - **1 node**: Please refer [here](./k8s_single_node.md) to deploy Seafile in a K8S single node instead a cluster. 2. If you have more available nodes for Seafile server, please provide them to the Seafile frontend service and **make sure there is only one backend service running**. Here is a simple relationship between the number of Seafile frontent services ($N_f$) and total nodes ($N_t$): $$ N_f = N_t - 1, @@ -73,7 +73,7 @@ For futher configuration details, you can refer [the official documents](https:/ ## Modify `seafile-env.yaml` and `seafile-secret.yaml` -Similar to Docker-base deployment, Seafile cluster in K8S deployment also supports use files to configure startup progress, you can modify common environment variables by +Similar to Docker-base deployment, Seafile cluster in K8S deployment also supports use files to configure startup progress, you can modify common [environment variables](./setup_pro_by_docker.md#downloading-and-modifying-env) by ```sh nano /opt/seafile-k8s-yaml/seafile-env.yaml @@ -179,7 +179,7 @@ Finally you can use the `tar -zcvf` and `tar -zxvf` commands to package the enti ``` !!! sucess - You can [view the pod's log](#container-management) to check the startup progress is normal or not, you will see the following message if server is running normally: + You can [view the pod's log](#container-management) to check the startup progress is normal or not. You can see the following message if server is running normally: ``` *** Running /etc/my_init.d/01_create_data_links.sh... @@ -210,7 +210,7 @@ Finally you can use the `tar -zcvf` and `tar -zxvf` commands to package the enti ## Container management -Similar to docker installation, you can also manage containers through [some kubectl commands](https://kubernetes.io/docs/reference/kubectl/#operations). For example, you can use the following command to check whether the relevant resources are started successfully and whether the relevant services can be accessed normally. First, execute the following command and remember the pod name with `seafile-` as the prefix (such as seafile-748b695648-d6l4g) +Similar to docker installation, you can also manage containers through [some kubectl commands](https://kubernetes.io/docs/reference/kubectl/#operations). For example, you can use the following command to check whether the relevant resources are started successfully and whether the relevant services can be accessed normally. First, execute the following command and remember the pod name with `seafile--` as the prefix (such as `seafile-frontend-748b695648-d6l4g`) ```shell kubectl get pods @@ -219,13 +219,13 @@ kubectl get pods You can check a status of a pod by ```shell -kubectl logs seafile-748b695648-d6l4g +kubectl logs seafile-frontend-748b695648-d6l4g ``` and enter a container by ```shell -kubectl exec -it seafile-748b695648-d6l4g -- bash +kubectl exec -it seafile-frontend-748b695648-d6l4g -- bash ``` ## Load balance and HTTPS diff --git a/manual/setup/k8s_single_node.md b/manual/setup/k8s_single_node.md new file mode 100644 index 00000000..d004d876 --- /dev/null +++ b/manual/setup/k8s_single_node.md @@ -0,0 +1,136 @@ +# Setup Seafile with a single K8S pod + +This manual explains how to deploy and run Seafile server on a Linux server using *Kubernetes* (***k8s*** thereafter) in a single pod (i.e., single node mode). So this document is essentially an extended description of the [Docker-based Seafile single-node deployment](./overview.md) (support both CE and Pro). + +For specific environment and configuration requirements, please refer to the description of the [Docker-based Seafile single-node deployment](./setup_pro_by_docker.md#requirements). Please also refer to the description of the ***K8S tool*** section in [here](./cluster_deploy_with_k8s.md#k8s-tools). + +## Gettings started + +For persisting data using in the docker-base deployment, `/opt/seafile-data`, is still adopted in this manual. What's more, all K8S YAML files will be placed in `/opt/seafile-k8s-yaml` (replace it when following these instructions if you would like to use another path). + +By the way, we don't provide the deployment methods of basic services (e.g., **Memcached**, **MySQL** and **Elasticsearch**) and seafile-compatibility components (e.g., **SeaDoc**) for K8S in our document. If you need to install these services in K8S format, ***you can refer to the rewrite method of this document.*** + +## Down load the YAML files for Seafile Server + +=== "Pro edition" + + ```sh + mkdir -p /opt/seafile-k8s-yaml + + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-deployment.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-persistentvolume.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-persistentvolumeclaim.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-service.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-env.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/pro/seafile-secret.yaml + ``` + +=== "Community edition" + + ```sh + mkdir -p /opt/seafile-k8s-yaml + + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-deployment.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-persistentvolume.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-persistentvolumeclaim.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-service.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-env.yaml + wget -P /opt/seafile-k8s-yaml https://manual.seafile.com/12.0/repo/k8s/ce/seafile-secret.yaml + ``` + +In here we suppose you download the YAML files in `/opt/seafile-k8s-yaml`, which mainly include about: + +- `seafile-deployment.yaml` for Seafile server pod management and creation, +- `seafile-service.yaml` for exposing Seafile services to the external network, +- `seafile-persistentVolume.yaml` for defining the location of a volume used for persistent storage on the host +- `seafile-persistentvolumeclaim.yaml` for declaring the use of persistent storage in the container. + +For futher configuration details, you can refer [the official documents](https://kubernetes.io/docs/tasks/configure-pod-container/). + +## Modify `seafile-env.yaml` and `seafile-secret.yaml` + +Similar to Docker-base deployment, Seafile cluster in K8S deployment also supports use files to configure startup progress, you can modify common [environment variables](./setup_pro_by_docker.md#downloading-and-modifying-env) by + +```sh +nano /opt/seafile-k8s-yaml/seafile-env.yaml +``` + +and sensitive information (e.g., password) by + +```sh +nano /opt/seafile-k8s-yaml/seafile-secret.yaml +``` + +!!! note "For `seafile-secret.yaml`" + To modify sensitive information (e.g., password), you need to convert the password into base64 encoding before writing it into the `seafile-secret.yaml` file: + + ```sh + echo -n '' | base64 + ``` + +!!! warning + For the fields marked with `<...>` are **required**, please make sure these items are filled in, otherwise Seafile server may not run properly. + +## Start Seafile server + +You can start Seafile server simply by + +```sh +kubectl apply -f /opt/seafile-k8s-yaml/ +``` + +!!! warning + By default, Seafile will access the ***Memcached*** and ***Elasticsearch*** with the specific service name: + + - ***Memcached***: `memcached` with port 11211 + - ***Elasticsearch***: `elasticsearch` with port 9200 + + If the above services are: + + - Not in your K8S pods (including using an external service) + - With different service name + - With different server port + + Please modfiy the files in `/opt/seafile-data/seafile/conf` to make correct the configurations for above services, otherwise the Seafile server cannot start normally. Then restart Seafile server: + + ```sh + kubectl delete -f /opt/seafile-k8s-yaml/ + kubectl apply -f /opt/seafile-k8s-yaml/ + ``` + +## Activating the Seafile License + +If you have a `seafile-license.txt` license file, simply put it in the volume of the Seafile container. The volumne's default path in the Compose file is `/opt/seafile-data`. If you have modified the path, save the license file under your custom path. + +!!! danger "If the license file has a different name or cannot be read, Seafile server will start with in trailer mode with most THREE users" + +Then restart Seafile: + +```bash +kubectl delete -f /opt/seafile-k8s-yaml/ +kubectl apply -f /opt/seafile-k8s-yaml/ +``` + +## Container management + +Similar to docker installation, you can also manage containers through [some kubectl commands](https://kubernetes.io/docs/reference/kubectl/#operations). For example, you can use the following command to check whether the relevant resources are started successfully and whether the relevant services can be accessed normally. First, execute the following command and remember the pod name with `seafile-` as the prefix (such as `seafile-748b695648-d6l4g`) + +```shell +kubectl get pods +``` + +You can check a status of a pod by + +```shell +kubectl logs seafile-748b695648-d6l4g +``` + +and enter a container by + +```shell +kubectl exec -it seafile-748b695648-d6l4g -- bash +``` + +## HTTPS + +Please refer [here](./cluster_deploy_with_k8s.md#load-balance-and-https) about suggestions of enabling HTTPS in K8S. diff --git a/manual/setup/migrate_backends_data.md b/manual/setup/migrate_backends_data.md index 85862a43..3fd9908d 100644 --- a/manual/setup/migrate_backends_data.md +++ b/manual/setup/migrate_backends_data.md @@ -1,203 +1,161 @@ ---- -status: new ---- - - # Migrate data between different backends -Seafile supports data migration between filesystem, s3, ceph, swift and Alibaba oss. - -Data migration takes 3 steps: - -1. Create a new temporary seafile.conf -2. Run migrate.sh to initially migrate objects -3. Run final migration -4. Replace the original seafile.conf +Seafile supports data migration between filesystem, s3, ceph, swift and Alibaba oss by a built-in script. Before migration, you have to ensure that **both S3 hosts can be accessed normally**. !!! warning "Migration from S3" Since version 11, when you migrate from S3 to other storage servers, you have to use V4 authentication protocol. This is because version 11 upgrades to Boto3 library, which fails to list objects from S3 when it's configured to use V2 authentication protocol. -## Create a new temporary seafile.conf +## Copy `seafile.conf` and use new S3 configurations -We need to add new backend configurations to this file (including `[block_backend]`, `[commit_object_backend]`, `[fs_object_backend]` options) and save it under a readable path. -Let's assume that we are migrating data to S3 and create temporary seafile.conf under `/opt` +During the migration process, Seafile needs to know where the data will be migrated to. The easiest way is to copy the original `seafile.conf` to a new path, and then use the new S3 configurations in this file. -``` -cat > seafile.conf << EOF -[commit_object_backend] -name = s3 -bucket = seacomm -key_id = ****** -key = ****** +=== "Deploy with Docker" -[fs_object_backend] -name = s3 -bucket = seafs -key_id = ****** -key = ****** + !!! warning + For deployment with Docker, the new `seafile.conf` has to **be put in the persistent directory** (e.g., `/opt/seafile-data/seafile.conf`) used by Seafile service. Otherwise the script cannot locate the new configurations file. -[block_backend] -name = s3 -bucket = seablk -key_id = ****** -key = ****** -EOF + ```sh + cp /opt/seafile-data/seafile/conf/seafile.conf /opt/seafile-data/seafile.conf -mv seafile.conf /opt + nano /opt/seafile-data/seafile.conf + ``` -``` +=== "Deploy from binary package" -If you want to migrate to a local file system, the seafile.conf temporary configuration example is as follows: + ```sh + cp /opt/seafile/conf/seafile.conf /opt/seafile.conf + + nano /opt/seafile.conf + ``` + +Then you can follow [here](./setup_with_s3.md) to use the new S3 configurations in the new `seafile.conf`. By the way, if you want to migrate to a local file system, the new `seafile.conf` configurations for S3 example is as follows: + +```conf +# ... other configurations -``` -cat > seafile.conf << EOF [commit_object_backend] name = fs -# the dir configuration is the new seafile-data path dir = /var/data_backup [fs_object_backend] name = fs -# the dir configuration is the new seafile-data path dir = /var/data_backup [block_backend] name = fs -# the dir configuration is the new seafile-data path dir = /var/data_backup - -EOF - -mv seafile.conf /opt - ``` -Repalce the configurations with your own choice. +## Stop Seafile Server -### Migrating to SSE-C Encrypted S3 Storage +Since the data migration process will not affect the operation of the Seafile service, if the original S3 data is operated during this process, the data may not be synchronized with the migrated data. Therefore, we recommend that you stop the Seafile service before executing the migration procedure. -If you are migrating to S3 storage, and want your data to be encrypted at rest, you can configure SSE-C encryption options in the temporary seafile.conf. Note that you have to use Seafile Pro 11 or newer and make sure your S3 storage supports SSE-C. +=== "Deploy with Docker" -``` -cat > seafile.conf << EOF -[commit_object_backend] -name = s3 -bucket = seacomm -key_id = ****** -key = ****** -use_v4_signature = true -use_https = true -sse_c_key = XiqMSf3x5ja4LRibBbV0sVntVpdHXl3P + ```sh + docker exec -it seafile bash + cd /opt/seafile/seafile-server-latest + ./seahub.sh stop + ./seafile.sh stop + ``` -[fs_object_backend] -name = s3 -bucket = seafs -key_id = ****** -key = ****** -use_v4_signature = true -use_https = true -sse_c_key = XiqMSf3x5ja4LRibBbV0sVntVpdHXl3P +=== "Deploy from binary package" -[block_backend] -name = s3 -bucket = seablk -key_id = ****** -key = ****** -use_v4_signature = true -use_https = true -sse_c_key = XiqMSf3x5ja4LRibBbV0sVntVpdHXl3P -EOF - -mv seafile.conf /opt - -``` - -`sse_c_key` is a string of 32 characters. - -You can generate `sse_c_key` with the following command： - -``` -openssl rand -base64 24 -``` - -## Migrating large number of objects - -If you have millions of objects in the storage (especially fs objects), it may take quite long time to migrate all objects. More than half of the time is spent on checking whether an object exists in the destination storage. **Since Pro edition 7.0.8**, a feature is added to speed-up the checking. - -Before running the migration script, please set this env variable: - -``` -export OBJECT_LIST_FILE_PATH=/path/to/object/list/file - -``` - -3 files will be created: `/path/to/object/list/file.commit`,`/path/to/object/list/file.fs`, `/path/to/object/list/file.blocks`. - -When you run the script for the first time, the object list file will be filled with existing objects in the destination. Then, when you run the script for the second time, it will load the existing object list from the file, instead of querying the destination. And newly migrated objects will also be added to the file. During migration, the migration process checks whether an object exists by checking the pre-loaded object list, instead of asking the destination, which will greatly speed-up the migration process. - -It's suggested that you don't interrupt the script during the "fetch object list" stage when you run it for the first time. Otherwise the object list in the file will be incomplete. - -Another trick to speed-up the migration is to increase the number of worker threads and size of task queue in the migration script. You can modify the `nworker` and `maxsize` variables in the following code: - -``` -class ThreadPool(object): - -def __init__(self, do_work, nworker=20): - self.do_work = do_work - self.nworker = nworker - self.task_queue = Queue.Queue(maxsize = 2000) - -``` - -The number of workers can be set to relatively large values, since they're mostly waiting for I/O operations to finished. - -## Decrypting encrypted storage backend - -If you have an encrypted storage backend (a deprecated feature no long supported now), you can use this script to migrate and decrypt the data from that backend to a new one. You can add the `--decrypt` option, which will decrypt the data while reading it, and then write the unencrypted data to the new backend. Note that you need add this option in all stages of the migration. - -``` -cd ~/haiwen/seafile-server-latest -./migrate.sh /opt --decrypt - -``` + ```sh + cd /opt/seafile/seafile-server-latest + ./seahub.sh stop + ./seafile.sh stop + ``` ## Run migrate.sh to initially migrate objects -This step will migrate **most of** objects from the source storage to the destination storage. You don't need to stop Seafile service at this stage as it may take quite long time to finish. Since the service is not stopped, some new objects may be added to the source storage during migration. Those objects will be handled in the next step. +This step will migrate **most of** objects from the source storage to the destination storage. You don't need to stop Seafile service at this stage as it may take quite long time to finish. Since the service is not stopped, some new objects may be added to the source storage during migration. Those objects will be handled in the next step: -We assume you have installed seafile pro server under `~/haiwen`, enter `~/haiwen/seafile-server-latest` and run migrate.sh with parent path of temporary seafile.conf as parameter, here is `/opt`. +!!! tip "Speed-up migrating large number of objects" + If you have millions of objects in the storage (especially the ***fs*** objects), it may take quite long time to migrate all objects and more than half is using to check whether an object exists in the destination storage. In this situation, you can modify the `nworker` and `maxsize` variables in the `migrate.py`: -``` -cd ~/haiwen/seafile-server-latest -./migrate.sh /opt + ```py + class ThreadPool(object): + def __init__(self, do_work, nworker=20): + self.do_work = do_work + self.nworker = nworker + self.task_queue = Queue.Queue(maxsize = 2000) + ``` -``` + However, if the two values (i.e., `nworker` and `maxsize`) ​​are too large, the improvement in data migration speed may not be obvious because the disk I/O bottleneck has been reached. -!!! tip - This script is completely reentrant. So you can stop and restart it, or run it many times. It will check whether an object exists in the destination before sending it. +!!! note "Encrypted storage backend data (deprecated)" + If you have an encrypted storage backend, you can use this script to migrate and decrypt the data from that backend to a new one. You can add the `--decrypt` option in calling the script, which will decrypt the data while reading it, and then write the unencrypted data to the new backend: -## Run final migration + ```sh + ./migrate.sh /opt --decrypt + ``` -New objects added during the last migration step will be migrated in this step. To prevent new objects being added, you have to stop Seafile service during the final migration operation. This usually take short time. If you have large number of objects, please following the optimization instruction in previous section. +=== "Deploy with Docker" -You just have to stop Seafile and Seahub service, then run the migration script again. + ```sh + # make sure you are in the container and in directory `/opt/seafile/seafile-server-latest` + ./migrate.sh /shared + + # exit container and stop it + exit + docker compose down + ``` -``` -cd ~/haiwen/seafile-server-latest -./migrate.sh /opt +=== "Deploy from binary package" -``` + ```sh + # make sure you are in the directory `/opt/seafile/seafile-server-latest` + ./migrate.sh /opt + ``` -## Replace the original seafile.conf +!!! success + You can see the following message if the migration process is done: -After running the script, we need replace the original seafile.conf with new one: + ``` + 2025-01-15 05:49:39,408 Start to fetch [commits] object from destination + 2025-01-15 05:49:39,422 Start to fetch [fs] object from destination + 2025-01-15 05:49:39,442 Start to fetch [blocks] object from destination + 2025-01-15 05:49:39,677 [commits] [0] objects exist in destination + 2025-01-15 05:49:39,677 Start to migrate [commits] object + 2025-01-15 05:49:39,749 [blocks] [0] objects exist in destination + 2025-01-15 05:49:39,755 Start to migrate [blocks] object + 2025-01-15 05:49:39,752 [fs] [0] objects exist in destination + 2025-01-15 05:49:39,762 Start to migrate [fs] object + 2025-01-15 05:49:40,602 Complete migrate [commits] object + 2025-01-15 05:49:40,626 Complete migrate [blocks] object + 2025-01-15 05:49:40,790 Complete migrate [fs] object + Done. + ``` -``` -mv /opt/seafile.conf ~/haiwen/conf +## Replace the original `seafile.conf` and start Seafile -``` +After running the script, we recommend that you check whether your data already exists on the new S3 storage backend server (i.e., the migration is successful, and the number and size of files should be the same). Then you can remove the file from the old S3 storage backend and replace the original `seafile.conf` from the new one: -now we only have configurations about backend, more config options, e.g. memcache and quota, can then be copied from the original seafile.conf file. +=== "Deploy with Docker" -After replacing seafile.conf, you can restart seafile server and access the data on the new backend. + ```sh + mv /opt/seafile-data/seafile.conf /opt/seafile-data/seafile/conf/seafile.conf + ``` +=== "Deploy from binary package" + + ```sh + mv /opt/seafile.conf /opt/seafile/conf/seafile.conf + ``` + +Finally, you can start Seafile server: + +=== "Deploy with Docker" + + ```sh + docker compose up -d + ``` + +=== "Deploy from binary package" + + ```sh + # make sure you are in the directory `/opt/seafile/seafile-server-latest` + ./seahub.sh start + ./seafile.sh start + ``` diff --git a/mkdocs.yml b/mkdocs.yml index b7d11575..fba65fa7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,6 +101,7 @@ nav: - Multiple Storage Backends: setup/setup_with_multiple_storage_backends.md - Data migration: setup/migrate_backends_data.md - Use SeaSearch as search engine (Pro): setup/use_seasearch.md + - Setup with Kubernetes (K8S, single pod mode): setup/k8s_single_node.md - Seafile Docker autostart: setup/seafile_docker_autostart.md - Deploy with an existing MySQL server: setup/setup_with_an_existing_mysql_server.md - Use other reverse proxy: setup/use_other_reverse_proxy.md