diff --git a/geninfo/genisolist.py b/geninfo/genisolist.py index 80121b5..21bc69a 100755 --- a/geninfo/genisolist.py +++ b/geninfo/genisolist.py @@ -8,6 +8,7 @@ import json import logging import collections import sys +import fnmatch from urllib.parse import urljoin from distutils.version import LooseVersion from configparser import ConfigParser @@ -21,6 +22,15 @@ parser.add_argument( '-d', '--dir', default=None, help='Override root directory.' ) +parser.add_argument( + '-R', '--remote', default=None, + help='[Remote Mode] Using rsync to get file list instead of reading from INI. Need the base of target site, for example, `mirror.nju.edu.cn`.' +) +parser.add_argument( + '-T', '--test', default=None, nargs="*", + help='Test specified `distro`s (multiple arguments input is supported) in INI. If Remote Mode is on, `distro`s must be specified in case of heavy rsync job.' +) + args = parser.parse_args() logger = logging.getLogger(__name__) @@ -64,7 +74,7 @@ def getSortKeys(template, result): return keys -def parseSection(items): +def parseSection(items, rsync=False): items = dict(items) if 'location' in items: @@ -78,13 +88,34 @@ def parseSection(items): pattern = items.get("pattern", "") prog = re.compile(pattern) - - images = [] images = {} - for location in locations: - logger.debug("[GLOB] %s", location) - for imagepath in glob.glob(location): + for location in locations: + + if rsync: + # find last non-wildcard path + non_regex = [] + for dir in location.split("/"): + if re.search("[\?\*\[\]]", dir): + break + non_regex.append(dir) + prefix = "/".join(non_regex + [""]) + rsync_url = args.remote.rstrip("/") + "/" + prefix + print("[RSync] %s" % rsync_url) + image_paths = rsyncQuery(rsync_url) + + # recover path + image_paths = [prefix + path for path in image_paths] + + # use fnmatch to mimic glob behavior + image_paths = list( + filter(lambda path: fnmatch.fnmatch(path, location), image_paths)) + + else: + logger.debug("[GLOB] %s", location) + image_paths = glob.glob(location) + + for imagepath in image_paths: logger.debug("[FILE] %s", imagepath) result = prog.search(imagepath) @@ -165,40 +196,74 @@ def getImageList(): if not (ini.read(CONFIG_FILE)): raise Exception("%s not found!" % CONFIG_FILE) - root = ini.get("%main%", 'root') - urlbase = ini.get("%main%", 'urlbase') - - if args.dir: - # Allow to override root in command-line - root = args.dir - prior = {} for (name, value) in ini.items("%main%"): if re.match("d\d+$", name): prior[value] = int(name[1:]) - oldcwd = os.getcwd() - os.chdir(root) - - img_dict = collections.defaultdict(list) - for section in ini.sections(): - if section == "%main%": - continue - for image in parseSection(ini.items(section)): - img_dict[image['distro']].append(image) - url_dict = {} - for distro, images in img_dict.items(): - images.sort(key=lambda x: x['sort_key'], reverse=True) - logger.debug("[IMAGES] %r %r", distro, images) - url_dict[distro] = [getDetail(image, urlbase) for image in images] + root = ini.get("%main%", 'root') + urlbase = ini.get("%main%", 'urlbase') - os.chdir(oldcwd) + if not args.remote: + # Local + + if args.dir: + # Allow to override root in command-line + root = args.dir + + oldcwd = os.getcwd() + os.chdir(root) + + img_dict = collections.defaultdict(list) + for section in ini.sections(): + if section == "%main%": + continue + for image in parseSection(ini.items(section)): + img_dict[image['distro']].append(image) + + for distro, images in img_dict.items(): + images.sort(key=lambda x: x['sort_key'], reverse=True) + logger.debug("[IMAGES] %r %r", distro, images) + url_dict[distro] = [getDetail(image, urlbase) for image in images] + + os.chdir(oldcwd) + + else: + # Remote + for spec_distro in args.test: + + img_dict = collections.defaultdict(list) + for section in ini.sections(): + if section == "%main%": + continue + if ini.get(section, 'distro') != spec_distro: + continue + else: + for image in parseSection(ini.items(section), rsync=True): + img_dict[image['distro']].append(image) + + for distro, images in img_dict.items(): + images.sort(key=lambda x: x['sort_key'], reverse=True) + logger.debug("[IMAGES] %r %r", distro, images) + url_dict[distro] = [getDetail(image, urlbase) + for image in images] return getJsonOutput(url_dict, prior) +def rsyncQuery(url: str): + rsync_proc = os.popen( + "rsync -r --list-only --no-motd rsync://%s | awk '{ $1=$2=$3=$4=\"\"; print substr($0,5); }'" % url) + return [path.rstrip() for path in rsync_proc.readlines()] + + if __name__ == "__main__": import sys logging.basicConfig(stream=sys.stderr, level=logging.WARNING) + if args.remote: + if not args.test: + raise ArgumentError( + None, 'If Remote Mode is on, images must be specified in case of heavy rsync job.') + print(getImageList())