15308555518/ tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-12-26 00:42:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

anaconda: create gziped meta json files 

Signed-off-by: Miao Wang <shankerwangmiao@gmail.com>
This commit is contained in:
Miao Wang 2023-02-22 02:34:16 +08:00
parent f9e6b2b2e9
commit 42315f6938
No known key found for this signature in database
GPG Key ID: B031E3CA7BFED96E
1 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
anaconda.py
View File

@ -3,6 +3,7 @@ import hashlib
import json
import logging
import os
import errno
import random
import shutil
import subprocess as sp
@ -74,6 +75,9 @@ EXCLUDED_PACKAGES = (
# connect and read timeout value
TIMEOUT_OPTION = (7, 10)
# Generate gzip archive for json files, size threshold
GEN_METADATA_JSON_GZIP_THRESHOLD = 1024 * 1024
logging.basicConfig(
level=logging.INFO,
format="[%(asctime)s] [%(levelname)s] %(message)s",
@ -190,12 +194,34 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
break
logging.error("Failed to download {}: {}".format(filename, err))
if os.path.getsize(tmp_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD:
sp.check_call(["gzip", "--no-name", "--keep", "--", str(tmp_repodata)])
shutil.move(str(tmp_repodata) + ".gz", str(local_dir / "repodata.json.gz"))
else:
# If the gzip file is not generated, remove the dangling gzip archive
try:
os.remove(str(local_dir / "repodata.json.gz"))
except OSError as e:
if e.errno != errno.ENOENT:
raise
shutil.move(str(tmp_repodata), str(local_dir / "repodata.json"))
shutil.move(str(tmp_bz2_repodata), str(local_dir / "repodata.json.bz2"))
tmp_current_repodata_gz_gened = False
if tmp_current_repodata.is_file():
if os.path.getsize(tmp_current_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD:
sp.check_call(["gzip", "--no-name", "--keep", "--", str(tmp_current_repodata)])
shutil.move(str(tmp_current_repodata) + ".gz", str(local_dir / "current_repodata.json.gz"))
tmp_current_repodata_gz_gened = True
shutil.move(str(tmp_current_repodata), str(
local_dir / "current_repodata.json"))
if not tmp_current_repodata_gz_gened:
try:
# If the gzip file is not generated, remove the dangling gzip archive
os.remove(str(local_dir / "current_repodata.json.gz"))
except OSError as e:
if e.errno != errno.ENOENT:
raise
if delete:
local_filelist = []