From ba6ace3b359dc665244a712c00247a6bc6fe01eb Mon Sep 17 00:00:00 2001 From: Xavier Yao Date: Sun, 6 Nov 2016 22:55:15 +0800 Subject: [PATCH] script and config to generate iso info --- geninfo/genisolist.ini | 178 +++++++++++++++++++++++++++++++++++++++++ geninfo/genisolist.py | 145 +++++++++++++++++++++++++++++++++ 2 files changed, 323 insertions(+) create mode 100644 geninfo/genisolist.ini create mode 100644 geninfo/genisolist.py diff --git a/geninfo/genisolist.ini b/geninfo/genisolist.ini new file mode 100644 index 0000000..7b0b740 --- /dev/null +++ b/geninfo/genisolist.ini @@ -0,0 +1,178 @@ +# This file is the config required by genisolist.py + +# This special section named "%main%" defined following variables: +# "root": HTTP root of mirrors. The script will locate the images in it. +# "urlbase": URL of mirrors prepended to image path. We want to use relative +# path, so set it to '/' instead of complete URL. +# "d[N]": For distribution sorting. where N is an positive integer. The value +# is disto name specified in below sections. Lower N makes the distro +# show higher. Default N is 0xFFFF for distro not mentioned. +[%main%] +root = /srv/www/ +urlbase = / +d10 = Ubuntu +d20 = Ubuntu 衍生版 +d30 = Debian +d40 = Arch Linux +d50 = Fedora +d60 = Deepin + +# Sections whose name isn't "%main%" defined a detect rule of image detection. +[archlinux] +# Section name is of no use, the display name is specified in "distro" option. +distro = Arch Linux +# listvers defined how many latest versions to display. +listvers = 1 +# "location" specifies globbing pathname of the image. The path is relative to +# the HTTP root (aka "root" in [%main%] section). Not all images match it is +# considered, you can use "pattern" option below to filter. +location = archlinux/iso/latest/archlinux-*-dual.iso +# "pattern" is a regular expression. If the pattern is found in image path +# found by "location", then the image is valid. Group capturing is to extract +# image info from image path name. +pattern = archlinux-(\d+\.\d+\.\d+)-(\w+).iso +# Following 3 options describes image info. "type" and "platform" is optional. +# $1, $2... here will be replaced by the string captured in above "pattern". +# Additionally, $0 will be replaced by the whole string matches the pattern. +# "version" is also used as the key to sort images of the same distro. +version = $1 +type = CLI-only +platform = $2 + +[ubuntu] +distro = Ubuntu +listvers = 4 +# If one glob is not enough for locating all images, you can use "location_N" +# to specify more globs. N start from 0 and must a sequence (0,1,2...). +location_0 = ubuntu-releases/[a-z]*/ubuntu-*-desktop-i386.iso +location_1 = ubuntu-releases/[a-z]*/ubuntu-*-desktop-amd64.iso +pattern = ubuntu-([0-9.]+)-desktop-(\w+).iso +version = $1 +type = LiveCD +platform = $2 + +# You can apply multiple rules (sections) for the same distro like following +# four sections. They will be shown in the same submenu on our web page. +[ubuntukylin] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/ubuntukylin/releases/[a-z]*/release/ubuntukylin-*-desktop-i386.iso +location_1 = ubuntu-cdimage/ubuntukylin/releases/[a-z]*/release/ubuntukylin-*-desktop-amd64.iso +pattern = ubuntukylin-([0-9.]+)-desktop-(\w+).iso +# Just a small trick. Constant string prefix won't affect sorting. +version = Ubuntu Kylin $1 +platform = $2 + +[kubuntu] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/kubuntu/releases/[a-z]*/release/kubuntu-*-desktop-i386.iso +location_1 = ubuntu-cdimage/kubuntu/releases/[a-z]*/release/kubuntu-*-desktop-amd64.iso +pattern = kubuntu-([0-9.]+)-desktop-(\w+).iso +version = Kubuntu $1 +platform = $2 + + +[lubuntu] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/lubuntu/releases/[a-z]*/release/lubuntu-*-desktop-i386.iso +location_1 = ubuntu-cdimage/lubuntu/releases/[a-z]*/release/lubuntu-*-desktop-amd64.iso +pattern = lubuntu-([0-9.]+)-desktop-(\w+).iso +version = Lubuntu $1 +platform = $2 + +[xubuntu] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/xubuntu/releases/[a-z]*/release/xubuntu-*-desktop-i386.iso +location_1 = ubuntu-cdimage/xubuntu/releases/[a-z]*/release/xubuntu-*-desktop-amd64.iso +pattern = xubuntu-([0-9.]+)-desktop-(\w+).iso +version = Xubuntu $1 +platform = $2 + +[ubuntu_gnome] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/ubuntu-gnome/releases/[a-z]*/release/ubuntu-gnome-*-desktop-i386.iso +location_1 = ubuntu-cdimage/ubuntu-gnome/releases/[a-z]*/release/ubuntu-gnome-*-desktop-amd64.iso +pattern = ubuntu-gnome-([0-9.]+)-desktop-(\w+).iso +version = Ubuntu Gnome $1 +platform = $2 + +[ubuntu_mate] +distro = Ubuntu 衍生版 +listvers = 1 +location_0 = ubuntu-cdimage/ubuntu-mate/releases/[a-z]*/release/ubuntu-mate-*-desktop-i386.iso +location_1 = ubuntu-cdimage/ubuntu-mate/releases/[a-z]*/release/ubuntu-mate-*-desktop-amd64.iso +pattern = ubuntu-mate-([0-9.]+)-desktop-(\w+).iso +version = Ubuntu Mate $1 +platform = $2 + +[debian_cd] +distro = Debian +listvers = 1 +location_0 = debian-cd/current/amd64/iso-cd/debian-[0-9]*-amd64-CD-1.iso +location_1 = debian-cd/current/i386/iso-cd/debian-[0-9]*-i386-CD-1.iso +pattern = debian-([0-9.]+)-(\w+)-CD-1.iso +version = $1 +type = CD installer +platform = $2 + +[debian_live] +distro = Debian +listvers = 1 +location_0 = debian-cd/current-live/amd64/iso-hybrid/debian-live-[0-9]*-amd64-*-desktop.iso +location_1 = debian-cd/current-live/i386/iso-hybrid/debian-live-[0-9]*-i386-*-desktop.iso +pattern = debian-live-([0-9.]+)-(\w+)-(\w+)-desktop.iso +version = $1 +type = $3 live +platform = $2 + +[centos] +distro = CentOS +listvers = 2 +location = centos/[0-9].*/isos/*/CentOS-[0-9]*.iso +pattern = CentOS-([0-9.]+)-(\w+)-([Mm]inimal|[Nn]et[Ii]nstall|Live\w+)(-[0-9]+|).iso +version = $1 +type = $3 +platform = $2 + +[fedora] +distro = Fedora +listvers = 2 +location_0 = fedora/releases/[1-9][0-9]/Workstation/*/iso/Fedora-Workstation-Live-*-[1-9][0-9]-*.iso +location_1 = fedora/releases/[1-9][0-9]/Spins/*/iso/Fedora-KDE-Live-*-[1-9][0-9]-*.iso +location_2 = fedora/releases/[1-9][0-9]/Spins/*/iso/Fedora-Xfce-Live-*-[1-9][0-9]-*.iso +pattern = Fedora-(Workstation|KDE|Xfce)-Live-(\w+)-(\d+)-.*\.iso +version = $3 +type = $1 +platform = $2 + +[opensuse_leap] +distro = openSUSE +listvers = 3 +location_0 = opensuse/distribution/leap/[0-9][0-9].[0-9]/iso/openSUSE-Leap-[0-9][0-9].[0-9]-DVD-*.iso +pattern = openSUSE-Leap-([0-9.]+)-DVD-(\w+).iso +version = $1 +platform = $2 + +[opensuse] +distro = openSUSE +listvers = 1 +location_0 = opensuse/distribution/[0-9][0-9].[0-9]/iso/openSUSE-[0-9][0-9].[0-9]-DVD-*.iso +location_1 = opensuse/distribution/[0-9][0-9].[0-9]/iso/openSUSE-[0-9][0-9].[0-9]-GNOME-Live-*.iso +location_2 = opensuse/distribution/[0-9][0-9].[0-9]/iso/openSUSE-[0-9][0-9].[0-9]-KDE-Live-*.iso +pattern = openSUSE-([0-9.]+)-(DVD|GNOME-Live|KDE-Live)-(\w+).iso +version = $1 +type = $2 +platform = $3 + + +[deepin] +distro = Deepin +listvers = 1 +location = deepin-cd/1*/*.iso +pattern = deepin-([0-9.]+)-(\w+).iso +version = $1 +platform = $2 diff --git a/geninfo/genisolist.py b/geninfo/genisolist.py new file mode 100644 index 0000000..4bcdf60 --- /dev/null +++ b/geninfo/genisolist.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import re +import glob +import json +import logging +from urllib.parse import urljoin +from distutils.version import LooseVersion +from configparser import ConfigParser + +logger = logging.getLogger(__name__) +CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'genisolist.ini') + + +def getPlatformPriority(platform): + platform = platform.lower() + if platform in ['amd64', 'x86_64', '64bit']: + return 100 + elif platform in ['i386', 'i486', 'i586', 'i686', 'x86', '32bit']: + return 90 + else: + return 0 + + +def parseSection(items): + items = dict(items) + + if 'location' in items: + locations = [items['location']] + else: + locations = [] + i = 0 + while ("location_%d" % i) in items: + locations.append(items["location_%d" % i]) + i += 1 + + pattern = items.get("pattern", "") + prog = re.compile(pattern) + + images = [] + for location in locations: + logger.debug("[GLOB] %s", location) + + for imagepath in glob.glob(location): + logger.debug("[FILE] %s", imagepath) + + result = prog.search(imagepath) + + if not(result): + logger.debug("[MATCH] None") + continue + else: + logger.debug("[MATCH] %r", result.groups()) + + group_count = len(result.groups()) + 1 + imageinfo = {"filepath": imagepath, "distro": items["distro"]} + + for prop in ("version", "type", "platform"): + s = items.get(prop, "") + for i in range(0, group_count): + s = s.replace("$%d" % i, result.group(i)) + imageinfo[prop] = s + + logger.debug("[JSON] %r", imageinfo) + images.append(imageinfo) + + #images.sort(key=lambda k: ( LooseVersion(k['version']), + images.sort(key=lambda k: (LooseVersion(k['version']), + getPlatformPriority(k['platform']), + k['type']), + reverse=True) + + i = 0 + versions = set() + listvers = int(items.get('listvers', 0xFF)) + for image in images: + versions.add(image['version']) + if len(versions) <= listvers: + yield image + else: + break + + +def getDescriptionAndURL(image_info, urlbase): + url = urljoin(urlbase, image_info['filepath']) + desc = "%s (%s%s)" % ( + image_info['version'], + image_info['platform'], + ", %s" % image_info['type'] if image_info['type'] else '' + ) + return (desc, url) + + +def getJsonOutput(url_dict, prio={}): + raw = [] + for distro in url_dict: + raw.append({ + "distro": distro, + "urls": [{"name": l[0], "url": l[1]} for l in url_dict[distro]] + }) + + raw.sort(key=lambda d: prio.get(d["distro"], 0xFFFF)) + + return json.dumps(raw) + + +def getImageList(): + ini = ConfigParser() + if not(ini.read(CONFIG_FILE)): + raise Exception("%s not found!" % CONFIG_FILE) + + root = ini.get("%main%", 'root') + urlbase = ini.get("%main%", 'urlbase') + + prior = {} + for (name, value) in ini.items("%main%"): + if re.match("d\d+$", name): + prior[value] = int(name[1:]) + + oldcwd = os.getcwd() + os.chdir(root) + + url_dict = {} + for section in ini.sections(): + if section == "%main%": + continue + for image in parseSection(ini.items(section)): + if not image['distro'] in url_dict: + url_dict[image['distro']] = [] + + url_dict[image['distro']].append( + getDescriptionAndURL(image, urlbase) + ) + + os.chdir(oldcwd) + + return getJsonOutput(url_dict, prior) + +if __name__ == "__main__": + import sys + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + print(getImageList()) +