From 42315f69387061d7267a08a1cfb781f5e64dbce6 Mon Sep 17 00:00:00 2001
From: Miao Wang <shankerwangmiao@gmail.com>
Date: Wed, 22 Feb 2023 02:34:16 +0800
Subject: [PATCH] anaconda: create gziped meta json files

Signed-off-by: Miao Wang <shankerwangmiao@gmail.com>
---
 anaconda.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/anaconda.py b/anaconda.py
index 6a86eac..9ba1d77 100755
--- a/anaconda.py
+++ b/anaconda.py
@@ -3,6 +3,7 @@ import hashlib
 import json
 import logging
 import os
+import errno
 import random
 import shutil
 import subprocess as sp
@@ -74,6 +75,9 @@ EXCLUDED_PACKAGES = (
 # connect and read timeout value
 TIMEOUT_OPTION = (7, 10)
 
+# Generate gzip archive for json files, size threshold
+GEN_METADATA_JSON_GZIP_THRESHOLD = 1024 * 1024
+
 logging.basicConfig(
     level=logging.INFO,
     format="[%(asctime)s] [%(levelname)s] %(message)s",
@@ -190,12 +194,34 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
                 break
             logging.error("Failed to download {}: {}".format(filename, err))
 
+    if os.path.getsize(tmp_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD:
+        sp.check_call(["gzip", "--no-name", "--keep", "--", str(tmp_repodata)])
+        shutil.move(str(tmp_repodata) + ".gz", str(local_dir / "repodata.json.gz"))
+    else:
+        # If the gzip file is not generated, remove the dangling gzip archive
+        try:
+            os.remove(str(local_dir / "repodata.json.gz"))
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
 
     shutil.move(str(tmp_repodata), str(local_dir / "repodata.json"))
     shutil.move(str(tmp_bz2_repodata), str(local_dir / "repodata.json.bz2"))
+    tmp_current_repodata_gz_gened = False
     if tmp_current_repodata.is_file():
+        if os.path.getsize(tmp_current_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD:
+            sp.check_call(["gzip", "--no-name", "--keep", "--", str(tmp_current_repodata)])
+            shutil.move(str(tmp_current_repodata) + ".gz", str(local_dir / "current_repodata.json.gz"))
+            tmp_current_repodata_gz_gened = True
         shutil.move(str(tmp_current_repodata), str(
             local_dir / "current_repodata.json"))
+    if not tmp_current_repodata_gz_gened:
+        try:
+            # If the gzip file is not generated, remove the dangling gzip archive
+            os.remove(str(local_dir / "current_repodata.json.gz"))
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
 
     if delete:
         local_filelist = []