15308555518/ mirror-web
1
0
Fork 0
mirror of https://github.com/tuna/mirror-web.git synced 2025-12-25 20:32:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

genisolist: format by black 

Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
Shengqi Chen 2024-05-03 14:07:54 +08:00
parent 6e03bff2a0
commit cd02c4e5d2
No known key found for this signature in database
GPG Key ID: 6EE389C0F18AF774
1 changed files with 87 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
geninfo/genisolist.py
View File

@ -16,6 +16,7 @@ from urllib.parse import urljoin
from configparser import ConfigParser from configparser import ConfigParser
from argparse import ArgumentParser, ArgumentError from argparse import ArgumentParser, ArgumentError
class Version: class Version:
def __init__(self, vstring=None): def __init__(self, vstring=None):
if vstring: if vstring:
@ -56,11 +57,11 @@ class Version:
class LooseVersion(Version): class LooseVersion(Version):
component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE)
def parse(self, vstring): def parse(self, vstring):
self.vstring = vstring self.vstring = vstring
components = [x for x in self.component_re.split(vstring) if x and x != '.'] components = [x for x in self.component_re.split(vstring) if x and x != "."]
for i, obj in enumerate(components): for i, obj in enumerate(components):
try: try:
components[i] = int(obj) components[i] = int(obj)
@ -102,42 +103,45 @@ class LooseVersion(Version):
if self_version > other_version: if self_version > other_version:
return 1 return 1
parser = ArgumentParser( parser = ArgumentParser(
prog='iso info list generator', prog="iso info list generator",
description='Generate iso info list', description="Generate iso info list",
)
parser.add_argument("-d", "--dir", default=None, help="Override root directory.")
parser.add_argument(
"-R",
"--remote",
default=None,
help="[Remote Mode] Using rsync to get file list instead of reading from INI. Need the base of target site, for example, `mirror.tuna.tsinghua.edu.cn`.",
) )
parser.add_argument( parser.add_argument(
'-d', '--dir', default=None, "-T",
help='Override root directory.' "--test",
) default=None,
parser.add_argument( nargs="*",
'-R', '--remote', default=None, help="Test specified `distro`s (multiple arguments input is supported) in INI. If Remote Mode is on, `distro`s must be specified in case of heavy rsync job.",
help='[Remote Mode] Using rsync to get file list instead of reading from INI. Need the base of target site, for example, `mirror.nju.edu.cn`.'
)
parser.add_argument(
'-T', '--test', default=None, nargs="*",
help='Test specified `distro`s (multiple arguments input is supported) in INI. If Remote Mode is on, `distro`s must be specified in case of heavy rsync job.'
) )
args = parser.parse_args() args = parser.parse_args()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'genisolist.ini') CONFIG_FILE = os.path.join(os.path.dirname(__file__), "genisolist.ini")
def getPlatformPriority(platform): def getPlatformPriority(platform):
platform = platform.lower() platform = platform.lower()
if platform in ['amd64', 'x86_64', '64bit']: if platform in ["amd64", "x86_64", "64bit"]:
return 100 return 100
elif platform in ['arm64', 'aarch64', 'arm64v8']: elif platform in ["arm64", "aarch64", "arm64v8"]:
return 95 return 95
elif platform in ['riscv64']: elif platform in ["riscv64"]:
return 95 return 95
elif platform in ['loongson2f', 'loongson3']: elif platform in ["loongson2f", "loongson3"]:
return 95 return 95
elif platform in ['i386', 'i486', 'i586', 'i686', 'x86', '32bit']: elif platform in ["i386", "i486", "i586", "i686", "x86", "32bit"]:
return 90 return 90
elif platform in ['arm32', 'armhf', 'armv7']: elif platform in ["arm32", "armhf", "armv7"]:
return 85 return 85
else: else:
return 0 return 0
@ -152,10 +156,10 @@ def renderTemplate(template, result):
def getSortKeys(template, result): def getSortKeys(template, result):
keys = [] keys = []
for i in template.split(' '): for i in template.split(" "):
if not i: if not i:
continue continue
if i[0] != '$': if i[0] != "$":
keys.append(i) keys.append(i)
else: else:
keys.append(result.group(int(i[1:])) or "") keys.append(result.group(int(i[1:])) or "")
@ -165,8 +169,8 @@ def getSortKeys(template, result):
def parseSection(items, rsync=False): def parseSection(items, rsync=False):
items = dict(items) items = dict(items)
if 'location' in items: if "location" in items:
locations = [items['location']] locations = [items["location"]]
else: else:
locations = [] locations = []
i = 0 i = 0
@ -179,7 +183,6 @@ def parseSection(items, rsync=False):
images = {} images = {}
for location in locations: for location in locations:
if rsync: if rsync:
# find last non-wildcard path # find last non-wildcard path
non_regex = [] non_regex = []
@ -197,7 +200,8 @@ def parseSection(items, rsync=False):
# use fnmatch to mimic glob behavior # use fnmatch to mimic glob behavior
image_paths = list( image_paths = list(
filter(lambda path: fnmatch.fnmatch(path, location), image_paths)) filter(lambda path: fnmatch.fnmatch(path, location), image_paths)
)
else: else:
logger.debug("[GLOB] %s", location) logger.debug("[GLOB] %s", location)
@ -218,14 +222,17 @@ def parseSection(items, rsync=False):
for prop in ("version", "type", "platform", "category"): for prop in ("version", "type", "platform", "category"):
imageinfo[prop] = renderTemplate(items.get(prop, ""), result) imageinfo[prop] = renderTemplate(items.get(prop, ""), result)
if 'version' not in imageinfo: if "version" not in imageinfo:
imageinfo['version'] = '0.0' imageinfo["version"] = "0.0"
sort_by = items.get("sort_by", "") sort_by = items.get("sort_by", "")
if not (sort_by): if not (sort_by):
imageinfo['sort_key'] = ( imageinfo["sort_key"] = (
imageinfo['version'], imageinfo['platform'], imageinfo['type']) imageinfo["version"],
imageinfo["platform"],
imageinfo["type"],
)
else: else:
imageinfo['sort_key'] = getSortKeys(sort_by, result) imageinfo["sort_key"] = getSortKeys(sort_by, result)
logger.debug("[JSON] %r", imageinfo) logger.debug("[JSON] %r", imageinfo)
key = renderTemplate(items.get("key_by", ""), result) key = renderTemplate(items.get("key_by", ""), result)
@ -234,17 +241,21 @@ def parseSection(items, rsync=False):
images[key].append(imageinfo) images[key].append(imageinfo)
for image_group in images.values(): for image_group in images.values():
if 'nosort' not in items: if "nosort" not in items:
image_group.sort(key=lambda k: (LooseVersion(k['version']), image_group.sort(
getPlatformPriority(k['platform']), key=lambda k: (
k['type']), LooseVersion(k["version"]),
reverse=True) getPlatformPriority(k["platform"]),
k["type"],
),
reverse=True,
)
i = 0 i = 0
versions = set() versions = set()
listvers = int(items.get('listvers', 0xFF)) listvers = int(items.get("listvers", 0xFF))
for image in image_group: for image in image_group:
versions.add(image['version']) versions.add(image["version"])
if len(versions) <= listvers: if len(versions) <= listvers:
yield image yield image
else: else:
@ -252,27 +263,34 @@ def parseSection(items, rsync=False):
def getDetail(image_info, urlbase): def getDetail(image_info, urlbase):
url = urljoin(urlbase, image_info['filepath']) url = urljoin(urlbase, image_info["filepath"])
desc = "%s (%s%s)" % ( desc = (
image_info['version'], "%s (%s%s)"
image_info['platform'], % (
", %s" % image_info['type'] if image_info['type'] else '' image_info["version"],
) if image_info['platform'] != "" else image_info['version'] image_info["platform"],
", %s" % image_info["type"] if image_info["type"] else "",
)
if image_info["platform"] != ""
else image_info["version"]
)
category = image_info.get('category', 'os') or "os" category = image_info.get("category", "os") or "os"
return (desc, url, category) return (desc, url, category)
def getJsonOutput(url_dict, prio={}): def getJsonOutput(url_dict, prio={}):
raw = [] raw = []
for distro in url_dict: for distro in url_dict:
raw.append({ raw.append(
"distro": distro, {
"category": list({c for _, _, c in url_dict[distro]})[0], "distro": distro,
"urls": [ "category": list({c for _, _, c in url_dict[distro]})[0],
{"name": name, "url": url} for name, url, _ in url_dict[distro] "urls": [
] {"name": name, "url": url} for name, url, _ in url_dict[distro]
}) ],
}
)
raw.sort(key=lambda d: prio.get(d["distro"], 0xFFFF)) raw.sort(key=lambda d: prio.get(d["distro"], 0xFFFF))
@ -285,13 +303,13 @@ def getImageList():
raise Exception("%s not found!" % CONFIG_FILE) raise Exception("%s not found!" % CONFIG_FILE)
prior = {} prior = {}
for (name, value) in ini.items("%main%"): for name, value in ini.items("%main%"):
if re.match(r"d\d+$", name): if re.match(r"d\d+$", name):
prior[value] = int(name[1:]) prior[value] = int(name[1:])
url_dict = {} url_dict = {}
root = ini.get("%main%", 'root') root = ini.get("%main%", "root")
urlbase = ini.get("%main%", 'urlbase') urlbase = ini.get("%main%", "urlbase")
if not args.remote: if not args.remote:
# Local # Local
@ -308,10 +326,10 @@ def getImageList():
if section == "%main%": if section == "%main%":
continue continue
for image in parseSection(ini.items(section)): for image in parseSection(ini.items(section)):
img_dict[image['distro']].append(image) img_dict[image["distro"]].append(image)
for distro, images in img_dict.items(): for distro, images in img_dict.items():
images.sort(key=lambda x: x['sort_key'], reverse=True) images.sort(key=lambda x: x["sort_key"], reverse=True)
logger.debug("[IMAGES] %r %r", distro, images) logger.debug("[IMAGES] %r %r", distro, images)
url_dict[distro] = [getDetail(image, urlbase) for image in images] url_dict[distro] = [getDetail(image, urlbase) for image in images]
@ -320,38 +338,41 @@ def getImageList():
else: else:
# Remote # Remote
for spec_distro in args.test: for spec_distro in args.test:
img_dict = collections.defaultdict(list) img_dict = collections.defaultdict(list)
for section in ini.sections(): for section in ini.sections():
if section == "%main%": if section == "%main%":
continue continue
if ini.get(section, 'distro') != spec_distro: if ini.get(section, "distro") != spec_distro:
continue continue
else: else:
for image in parseSection(ini.items(section), rsync=True): for image in parseSection(ini.items(section), rsync=True):
img_dict[image['distro']].append(image) img_dict[image["distro"]].append(image)
for distro, images in img_dict.items(): for distro, images in img_dict.items():
images.sort(key=lambda x: x['sort_key'], reverse=True) images.sort(key=lambda x: x["sort_key"], reverse=True)
logger.debug("[IMAGES] %r %r", distro, images) logger.debug("[IMAGES] %r %r", distro, images)
url_dict[distro] = [getDetail(image, urlbase) url_dict[distro] = [getDetail(image, urlbase) for image in images]
for image in images]
return getJsonOutput(url_dict, prior) return getJsonOutput(url_dict, prior)
def rsyncQuery(url: str): def rsyncQuery(url: str):
rsync_proc = os.popen( rsync_proc = os.popen(
"rsync -r --list-only --no-motd rsync://%s | awk '{ $1=$2=$3=$4=\"\"; print substr($0,5); }'" % url) "rsync -r --list-only --no-motd rsync://%s | awk '{ $1=$2=$3=$4=\"\"; print substr($0,5); }'"
% url
)
return [path.rstrip() for path in rsync_proc.readlines()] return [path.rstrip() for path in rsync_proc.readlines()]
if __name__ == "__main__": if __name__ == "__main__":
import sys import sys
logging.basicConfig(stream=sys.stderr, level=logging.WARNING) logging.basicConfig(stream=sys.stderr, level=logging.WARNING)
if args.remote: if args.remote:
if not args.test: if not args.test:
raise ArgumentError( raise ArgumentError(
None, 'If Remote Mode is on, images must be specified in case of heavy rsync job.') None,
"If Remote Mode is on, images must be specified in case of heavy rsync job.",
)
print(getImageList()) print(getImageList())