From b8b7bb76e423c33ab161977c8645186af1668944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=81=A5=E8=BE=89?= <40563566+mrwangjianhui@users.noreply.github.com> Date: Thu, 30 Dec 2021 14:22:14 +0800 Subject: [PATCH] update elasticsearch --- ...d_and_setup_seafile_professional_server.md | 21 +- .../deploy_seafile_cluster_with_docker.md | 4 +- .../pro-edition/7.1_8.0/docker-compose.yml | 62 ++++ .../deploy_seafile_pro_with_docker.md | 12 +- manual/docker/pro-edition/docker-compose.yml | 6 +- manual/upgrade/upgrade_notes_for_9.0.x.md | 269 +++++++++++++++++- 6 files changed, 349 insertions(+), 25 deletions(-) create mode 100644 manual/docker/pro-edition/7.1_8.0/docker-compose.yml diff --git a/manual/deploy_pro/download_and_setup_seafile_professional_server.md b/manual/deploy_pro/download_and_setup_seafile_professional_server.md index b4518a0d..9ed59a16 100644 --- a/manual/deploy_pro/download_and_setup_seafile_professional_server.md +++ b/manual/deploy_pro/download_and_setup_seafile_professional_server.md @@ -410,7 +410,7 @@ Now you can access Seafile via the web interface at the host address and port 80 ## Configing ElasticSearch * For Seafile PE 8.0.x and previous versions, the Seafile installation package already includes ElasticSearch, you can directly use it. -* For Seafile PE 9.0.x and later versions, ElasticSearch needs to be installed and maintained separately (Due to copyright reasons, ElasticSearch 6.8.x cannot be brought into the Seafile package) +* For Seafile PE 9.0.x and later versions, ElasticSearch needs to be installed and maintained separately (Due to copyright reasons, ElasticSearch 7.x cannot be brought into the Seafile package) ### ElasticSearch Deployment @@ -418,7 +418,7 @@ We use Docker to deploy ElasticSearch as an example, so you need to install Dock ``` ## Pull ElasticSearch Image -docker pull elasticsearch:6.8.20 +docker pull elasticsearch:7.16.2 ``` ``` @@ -429,17 +429,16 @@ mkdir -p /opt/seafile-elasticsearch/data && chmod -R 777 /opt/seafile-elasticse ``` ## Start ElasticSearch Container docker run -d \ - -p 9200:9200 \ - -e "discovery.type=single-node" \ - -e "bootstrap.memory_lock=true" \ - -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \ - -v /opt/seafile-elasticsearch/data:/usr/share/elasticsearch/data \ - --name es \ - --restart=always \ - elasticsearch:6.8.20 +--name es \ +-p 9200:9200 \ +-e "discovery.type=single-node" -e "bootstrap.memory_lock=true" \ +-e "ES_JAVA_OPTS=-Xms1g -Xmx1g" -e "xpack.security.enabled=false" \ +--restart=always \ +-v /opt/seafile-elasticsearch/data:/usr/share/elasticsearch/data \ +-d elasticsearch:7.16.2 ``` -**NOTE**：seafile PE 9.0.x only supports ElasticSearch 6.8.x version. +**NOTE**：seafile PE 9.0.x only supports ElasticSearch 7.x version, it’s best to use 7.16.2 and above. ### Seafile Configuration diff --git a/manual/docker/cluster/deploy_seafile_cluster_with_docker.md b/manual/docker/cluster/deploy_seafile_cluster_with_docker.md index c4b0f062..785d074b 100644 --- a/manual/docker/cluster/deploy_seafile_cluster_with_docker.md +++ b/manual/docker/cluster/deploy_seafile_cluster_with_docker.md @@ -83,7 +83,7 @@ $ vim docker-compose.yml version: '2.0' services: seafile: - image: docker.seafile.top/seafileltd/seafile-pro-mc:9.0.2 + image: docker.seafile.top/seafileltd/seafile-pro-mc:latest container_name: seafile ports: - 80:80 @@ -223,7 +223,7 @@ $ vim docker-compose.yml version: '2.0' services: seafile: - image: docker.seafile.top/seafileltd/seafile-pro-mc:9.0.2 + image: docker.seafile.top/seafileltd/seafile-pro-mc:latest container_name: seafile ports: - 80:80 diff --git a/manual/docker/pro-edition/7.1_8.0/docker-compose.yml b/manual/docker/pro-edition/7.1_8.0/docker-compose.yml new file mode 100644 index 00000000..f2acba07 --- /dev/null +++ b/manual/docker/pro-edition/7.1_8.0/docker-compose.yml @@ -0,0 +1,62 @@ +version: '2.0' +services: + db: + image: mariadb:10.5 + container_name: seafile-mysql + environment: + - MYSQL_ROOT_PASSWORD=db_dev # Requested, set the root's password of MySQL service. + - MYSQL_LOG_CONSOLE=true + volumes: + - /opt/seafile-mysql/db:/var/lib/mysql # Requested, specifies the path to MySQL data persistent store. + networks: + - seafile-net + + memcached: + image: memcached:1.5.6 + container_name: seafile-memcached + entrypoint: memcached -m 256 + networks: + - seafile-net + + elasticsearch: + image: seafileltd/elasticsearch-with-ik:5.6.16 + container_name: seafile-elasticsearch + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 2g + volumes: + - /opt/seafile-elasticsearch/data:/usr/share/elasticsearch/data # Requested, specifies the path to Elasticsearch data persistent store. + networks: + - seafile-net + + seafile: + image: docker.seafile.top/seafileltd/seafile-pro-mc:latest + container_name: seafile + ports: + - "80:80" +# - "443:443" # If https is enabled, cancel the comment. + volumes: + - /opt/seafile-data:/shared # Requested, specifies the path to Seafile data persistent store. + environment: + - DB_HOST=db + - DB_ROOT_PASSWD=db_dev # Requested, the value shuold be root's password of MySQL service. +# - TIME_ZONE=Asia/Shanghai # Optional, default is UTC. Should be uncomment and set to your local time zone. + - SEAFILE_ADMIN_EMAIL=me@example.com # Specifies Seafile admin user, default is 'me@example.com' + - SEAFILE_ADMIN_PASSWORD=asecret # Specifies Seafile admin password, default is 'asecret' + - SEAFILE_SERVER_LETSENCRYPT=false # Whether to use https or not + - SEAFILE_SERVER_HOSTNAME=example.seafile.com # Specifies your host name if https is enabled + depends_on: + - db + - memcached + - elasticsearch + networks: + - seafile-net + +networks: + seafile-net: diff --git a/manual/docker/pro-edition/deploy_seafile_pro_with_docker.md b/manual/docker/pro-edition/deploy_seafile_pro_with_docker.md index cc70f7a6..e4ab87bf 100644 --- a/manual/docker/pro-edition/deploy_seafile_pro_with_docker.md +++ b/manual/docker/pro-edition/deploy_seafile_pro_with_docker.md @@ -25,11 +25,15 @@ docker pull {host}/seafileltd/seafile-pro-mc:latest ``` -You can find the private registry information on the [customer center download page](https://customer.seafile.com/downloads/). +You can find the private registry information on the [customer center download page](https://customer.seafile.com/downloads/). ### Download and modify docker-compose.yml -Download [docker-compose.yml](https://download.seafile.com/d/320e8adf90fa43ad8fee/files/?p=/docker/pro-edition/docker-compose.yml) sample file to your host. Then modify the file according to your environtment. The following fields are needed to be modified: +Seafile 7.1 to 8.0 version download URL: [docker-compose.yml](./7.1_8.0/docker-compose.yml) + +Seafile 9.0 and later version download URL: [docker-compose.yml](./docker-compose.yml) + +Then modify the file according to your environtment. The following fields are needed to be modified: * The password of MySQL root (MYSQL_ROOT_PASSWORD and DB_ROOT_PASSWD) * The volume directory of MySQL data (volumes) @@ -51,7 +55,7 @@ Wait for a few minutes for the first time initialization, then visit `http://sea ### Put your licence file(seafile-license.txt) -If you have a `seafile-license.txt` licence file, simply put it in the volume directory of Seafile data. If the directory is `/opt/seafile-data` So, in your host machine: +If you have a `seafile-license.txt` licence file, simply put it in the volume directory of Seafile data. If the directory is `/opt/seafile-data` So, in your host machine: ``` cp /path/to/seafile-license.txt /opt/seafile-data/seafile/ @@ -135,7 +139,7 @@ server { ### Modify Seafile server configurations -The config files are under `shared/seafile/conf`. You can modify the configurations according to [Seafile manual](https://manual.seafile.com/) +The config files are under `shared/seafile/conf`. You can modify the configurations according to [Seafile manual](https://manual.seafile.com/) After modification, you need to restart the container: diff --git a/manual/docker/pro-edition/docker-compose.yml b/manual/docker/pro-edition/docker-compose.yml index 7f3a8ddb..67e067e0 100644 --- a/manual/docker/pro-edition/docker-compose.yml +++ b/manual/docker/pro-edition/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.0' services: db: - image: mariadb:10.1 + image: mariadb:10.5 container_name: seafile-mysql environment: - MYSQL_ROOT_PASSWORD=db_dev # Requested, set the root's password of MySQL service. @@ -19,7 +19,7 @@ services: - seafile-net elasticsearch: - image: seafileltd/elasticsearch-with-ik:5.6.16 + image: elasticsearch:7.16.2 container_name: seafile-elasticsearch environment: - discovery.type=single-node @@ -36,7 +36,7 @@ services: - seafile-net seafile: - image: docker.seadrive.org/seafileltd/seafile-pro-mc:latest + image: docker.seafile.top/seafileltd/seafile-pro-mc:latest container_name: seafile ports: - "80:80" diff --git a/manual/upgrade/upgrade_notes_for_9.0.x.md b/manual/upgrade/upgrade_notes_for_9.0.x.md index 343f1af4..76b505ed 100644 --- a/manual/upgrade/upgrade_notes_for_9.0.x.md +++ b/manual/upgrade/upgrade_notes_for_9.0.x.md @@ -52,21 +52,92 @@ sudo pip3 install pycryptodome==3.12.0 cffi==1.14.0 3. Start Seafile-9.0.x server. -## Update ElasticSearch +### Update ElasticSearch -Download ElasticSearch image： +#### Method one, rebuild the index and discard the old index data + +If your elasticsearch data is not large, it is recommended to deploy the latest 7.x version of ElasticSearch and then rebuild the new index. Specific steps are as follows + +Download ElasticSearch image + +``` +docker pull elasticsearch:7.16.2 + +``` + +Create a new folder to store ES data and give the folder permissions + +``` +mkdir -p /opt/seafile-elasticsearch/data && chmod -R 777 /opt/seafile-elasticsearch/data/ + +``` + +Start ES docker image + +``` +docker run -d \ +--name es \ +-p 9200:9200 \ +-e "discovery.type=single-node" -e "bootstrap.memory_lock=true" \ +-e "ES_JAVA_OPTS=-Xms1g -Xmx1g" -e "xpack.security.enabled=false" \ +--restart=always \ +-v /opt/seafile-elasticsearch/data:/usr/share/elasticsearch/data \ +-d elasticsearch:7.16.2 + +``` + +Delete old index data + +``` +rm -rf /opt/seafile/pro-data/search/data/* + +``` + +Modify seafevents.conf + +``` +[INDEX FILES] +external_es_server = true +es_host = your server's IP +es_port = 9200 + +``` + +Restart seafile + +``` +su seafile +cd seafile-server-latest/ +./seafile.sh stop && ./seahub.stop +./seafile.sh start && ./seahub.start + +``` + +#### Method two, reindex the existing data + +If your data volume is relatively large, it will take a long time to rebuild indexes for all Seafile databases, so you can reindex the existing data. This requires the following steps + +* Download and start ElasticSearch 6.8.20 +* Use the existing data to execute ElasticSearch Reindex in order to build an index that can be used in 7.x +* Download and start ElasticSearch 7.x + +The detailed process is as follows + +Download ElasticSearch image: ``` docker pull elasticsearch:6.8.20 + ``` -Create a new folder to store ES data +PS：For seafile version 9.0, you need to manually create the elasticsearch mapping path on the host machine and give it 777 permission, otherwise elasticsearch will report path permission problems when starting, the command is as follows ``` mkdir -p /opt/seafile-elasticsearch/data + ``` -Move original data to the new folder +Move original data to the new folder and give the folder permissions ``` mv /opt/seafile/pro-data/search/data/* /opt/seafile-elasticsearch/data/ @@ -82,13 +153,196 @@ docker run -d --name es -p 9200:9200 -e "discovery.type=single-node" -e "bootst Note:`ES_JAVA_OPTS` can be adjusted according to your need. -Modify seafevents.conf （`external_es_server` need to be changed to true） +Create an index with 7.x compatible mappings. + +``` +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repo_head?include_type_name=false&pretty=true' -d ' +{ + "mappings" : { + "properties" : { + "commit" : { + "type" : "text", + "index" : false + }, + "repo" : { + "type" : "text", + "index" : false + }, + "updatingto" : { + "type" : "text", + "index" : false + } + } + } +}' + +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repofiles/?include_type_name=false&pretty=true' -d ' +{ + "settings" : { + "index" : { + "analysis" : { + "analyzer" : { + "seafile_file_name_ngram_analyzer" : { + "filter" : [ + "lowercase" + ], + "type" : "custom", + "tokenizer" : "seafile_file_name_ngram_tokenizer" + } + }, + "tokenizer" : { + "seafile_file_name_ngram_tokenizer" : { + "type" : "ngram", + "min_gram" : "3", + "max_gram" : "4" + } + } + } + } + }, + "mappings" : { + "properties" : { + "content" : { + "type" : "text", + "term_vector" : "with_positions_offsets" + }, + "filename" : { + "type" : "text", + "fields" : { + "ngram" : { + "type" : "text", + "analyzer" : "seafile_file_name_ngram_analyzer" + } + } + }, + "is_dir" : { + "type" : "boolean", + "index" : false + }, + "mtime" : { + "type" : "date" + }, + "path" : { + "type" : "keyword" + }, + "repo" : { + "type" : "keyword" + }, + "size" : { + "type" : "long" + }, + "suffix" : { + "type" : "keyword" + } + } + } +}' + +``` + +Set the `refresh_interval` to `-1` and the `number_of_replicas` to `0` for efficient reindexing: + +``` +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repo_head/_settings?pretty' -d ' +{ + "index" : { + "refresh_interval" : "-1", + "number_of_replicas" : 0 + } +}' + +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repofiles/_settings?pretty' -d ' +{ + "index" : { + "refresh_interval" : "-1", + "number_of_replicas" : 0 + } +}' + +``` + +Use the [reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/docs-reindex.html) to copy documents from the 5.x index into the new index. + +``` +curl -X POST -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/_reindex/?pretty' -d ' +{ + "source": { + "index": "repo_head", + "type": "repo_commit" + }, + "dest": { + "index": "new_repo_head", + "type": "_doc" + } +}' + +curl -X POST -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/_reindex/?pretty' -d ' +{ + "source": { + "index": "repofiles", + "type": "file" + }, + "dest": { + "index": "new_repofiles", + "type": "_doc" + } +}' + +``` + +Reset the `refresh_interval` and `number_of_replicas` to the values used in the old index. + +``` +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repo_head/_settings?pretty' -d ' +{ + "index" : { + "refresh_interval" : null, + "number_of_replicas" : 1 + } +}' + +curl -X PUT -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/new_repofiles/_settings?pretty' -d ' +{ + "index" : { + "refresh_interval" : null, + "number_of_replicas" : 1 + } +}' + +``` + +Wait for the index status to change to `green`. + +``` +curl http{s}://{es server IP}:9200/_cluster/health?pretty + +``` + +Use the [aliases API](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-aliases.html) delete the old index and add an alias with the old index name to the new index. + +``` +curl -X POST -H 'Content-Type: application/json' 'http{s}://{es server IP}:9200/_aliases?pretty' -d ' +{ + "actions": [ + {"remove_index": {"index": "repo_head"}}, + {"remove_index": {"index": "repofiles"}}, + {"add": {"index": "new_repo_head", "alias": "repo_head"}}, + {"add": {"index": "new_repofiles", "alias": "repofiles"}} + ] +}' + +``` + +After reindex, modify the configuration in Seafile. + +Modify seafevents.conf ``` [INDEX FILES] external_es_server = true es_host = your server's IP es_port = 9200 + ``` Restart seafile @@ -98,4 +352,9 @@ su seafile cd seafile-server-latest/ ./seafile.sh stop && ./seahub.stop ./seafile.sh start && ./seahub.start + ``` + +#### Method three, if you are in a cluster environment + +Deploy a new ElasticSeach 7.x service, use Seafile 9.0 version to deploy a new backend node, and connect to ElasticSeach 7.x. The background node does not start the Seafile background service, just manually run the command `./pro/pro.py search --update`, and then upgrade the other nodes to Seafile 9.0 version and use the new ElasticSeach 7.x after the index is created. Then deactivate the old backend node and the old version of ElasticSeach.