genisoinfo: add rsync remote option to test locally

geninfo/genisolist.py

@@ -8,6 +8,7 @@ import json
 import logging
 import collections
 import sys
+import fnmatch
 from urllib.parse import urljoin
 from distutils.version import LooseVersion
 from configparser import ConfigParser
@@ -21,6 +22,15 @@ parser.add_argument(
     '-d', '--dir', default=None,
     help='Override root directory.'
 )
+parser.add_argument(
+    '-R', '--remote', default=None,
+    help='[Remote Mode] Using rsync to get file list instead of reading from INI. Need the base of target site, for example, `mirror.nju.edu.cn`.'
+)
+parser.add_argument(
+    '-T', '--test', default=None, nargs="*",
+    help='Test specified `distro`s (multiple arguments input is supported) in INI. If Remote Mode is on, `distro`s must be specified in case of heavy rsync job.'
+)
+
 args = parser.parse_args()
 
 logger = logging.getLogger(__name__)
@@ -64,7 +74,7 @@ def getSortKeys(template, result):
     return keys
 
 
-def parseSection(items):
+def parseSection(items, rsync=False):
     items = dict(items)
 
     if 'location' in items:
@@ -78,13 +88,34 @@ def parseSection(items):
 
     pattern = items.get("pattern", "")
     prog = re.compile(pattern)
-
-    images = []
     images = {}
-    for location in locations:
-        logger.debug("[GLOB] %s", location)
 
-        for imagepath in glob.glob(location):
+    for location in locations:
+
+        if rsync:
+            # find last non-wildcard path
+            non_regex = []
+            for dir in location.split("/"):
+                if re.search("[\?\*\[\]]", dir):
+                    break
+                non_regex.append(dir)
+            prefix = "/".join(non_regex + [""])
+            rsync_url = args.remote.rstrip("/") + "/" + prefix
+            print("[RSync] %s" % rsync_url)
+            image_paths = rsyncQuery(rsync_url)
+
+            # recover path
+            image_paths = [prefix + path for path in image_paths]
+
+            # use fnmatch to mimic glob behavior
+            image_paths = list(
+                filter(lambda path: fnmatch.fnmatch(path, location), image_paths))
+
+        else:
+            logger.debug("[GLOB] %s", location)
+            image_paths = glob.glob(location)
+
+        for imagepath in image_paths:
             logger.debug("[FILE] %s", imagepath)
 
             result = prog.search(imagepath)
@@ -165,40 +196,74 @@ def getImageList():
     if not (ini.read(CONFIG_FILE)):
         raise Exception("%s not found!" % CONFIG_FILE)
 
-    root = ini.get("%main%", 'root')
-    urlbase = ini.get("%main%", 'urlbase')
-
-    if args.dir:
-        # Allow to override root in command-line
-        root = args.dir
-
     prior = {}
     for (name, value) in ini.items("%main%"):
         if re.match("d\d+$", name):
             prior[value] = int(name[1:])
 
-    oldcwd = os.getcwd()
-    os.chdir(root)
-
-    img_dict = collections.defaultdict(list)
-    for section in ini.sections():
-        if section == "%main%":
-            continue
-        for image in parseSection(ini.items(section)):
-            img_dict[image['distro']].append(image)
-
     url_dict = {}
-    for distro, images in img_dict.items():
-        images.sort(key=lambda x: x['sort_key'], reverse=True)
-        logger.debug("[IMAGES] %r %r", distro, images)
-        url_dict[distro] = [getDetail(image, urlbase) for image in images]
+    root = ini.get("%main%", 'root')
+    urlbase = ini.get("%main%", 'urlbase')
 
-    os.chdir(oldcwd)
+    if not args.remote:
+        # Local
+
+        if args.dir:
+            # Allow to override root in command-line
+            root = args.dir
+
+        oldcwd = os.getcwd()
+        os.chdir(root)
+
+        img_dict = collections.defaultdict(list)
+        for section in ini.sections():
+            if section == "%main%":
+                continue
+            for image in parseSection(ini.items(section)):
+                img_dict[image['distro']].append(image)
+
+        for distro, images in img_dict.items():
+            images.sort(key=lambda x: x['sort_key'], reverse=True)
+            logger.debug("[IMAGES] %r %r", distro, images)
+            url_dict[distro] = [getDetail(image, urlbase) for image in images]
+
+        os.chdir(oldcwd)
+
+    else:
+        # Remote
+        for spec_distro in args.test:
+
+            img_dict = collections.defaultdict(list)
+            for section in ini.sections():
+                if section == "%main%":
+                    continue
+                if ini.get(section, 'distro') != spec_distro:
+                    continue
+                else:
+                    for image in parseSection(ini.items(section), rsync=True):
+                        img_dict[image['distro']].append(image)
+
+            for distro, images in img_dict.items():
+                images.sort(key=lambda x: x['sort_key'], reverse=True)
+                logger.debug("[IMAGES] %r %r", distro, images)
+                url_dict[distro] = [getDetail(image, urlbase)
+                                    for image in images]
 
     return getJsonOutput(url_dict, prior)
 
 
+def rsyncQuery(url: str):
+    rsync_proc = os.popen(
+        "rsync -r --list-only --no-motd rsync://%s | awk '{ $1=$2=$3=$4=\"\"; print substr($0,5); }'" % url)
+    return [path.rstrip() for path in rsync_proc.readlines()]
+
+
 if __name__ == "__main__":
     import sys
     logging.basicConfig(stream=sys.stderr, level=logging.WARNING)
+    if args.remote:
+        if not args.test:
+            raise ArgumentError(
+                None, 'If Remote Mode is on, images must be specified in case of heavy rsync job.')
+
     print(getImageList())