remove robots.txt

gen_desc.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import urllib.request
-from collections import OrderedDict
-
-tunasync = json.loads(
-    urllib.request
-    .urlopen("https://mirrors.tuna.tsinghua.edu.cn/static/tunasync.json")
-    .read()
-    .decode('utf-8')
-)
-
-mirrors = list({t['name'] for t in tunasync})
-mirrors.sort()
-
-with open('_data/mirror_desc.json') as f:
-    descriptions = json.load(f)
-
-descriptions = {
-    x['name']: x['desc'] for x in descriptions
-}
-for m in mirrors:
-    if m not in descriptions:
-        descriptions[m] = ""
-
-descriptions = sorted(
-    [
-        OrderedDict([('name', name), ('desc', desc)])
-        for name, desc in descriptions.items()
-    ],
-    key=lambda x: x['name']
-)
-
-content = (
-    json
-    .dumps(descriptions, indent=2)
-    .encode('utf-8')
-    .decode('unicode-escape')
-    .encode('utf-8')
-)
-
-with open('_data/mirror_desc.json', 'wb') as f:
-    f.write(content)
-
-# vim: ts=4 sw=4 sts=4 expandtab

gen_robot.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-HOST="mirrors.tuna.tsinghua.edu.cn"
+HOST=${1:-"mirrors.tuna.tsinghua.edu.cn"}
 
 echo "# robots.txt for https://$HOST" > robots.txt
 echo 'User-agent: *' >> robots.txt

robots.txt

@@ -1,173 +0,0 @@
-# robots.txt for https://mirrors.tuna.tsinghua.edu.cn
-User-agent: *
-
-Disallow: /logs
-Disallow: /AOSP
-Disallow: /CPAN
-Disallow: /CRAN
-Disallow: /CTAN
-Disallow: /CocoaPods
-Disallow: /ELK
-Disallow: /HHVM
-Disallow: /NetBSD
-Disallow: /OpenBSD
-Disallow: /adobe-fonts
-Disallow: /alpine
-Disallow: /anaconda
-Disallow: /anthon
-Disallow: /aosp-monthly
-Disallow: /apache
-Disallow: /arch4edu
-Disallow: /archlinux
-Disallow: /archlinuxarm
-Disallow: /archlinuxcn
-Disallow: /armbian-releases
-Disallow: /armbian
-Disallow: /artixlinux-iso
-Disallow: /artixlinux
-Disallow: /bananian
-Disallow: /binutils-gdb.git
-Disallow: /bioconductor
-Disallow: /bjlx
-Disallow: /blackarch
-Disallow: /cc
-Disallow: /centos-altarch
-Disallow: /centos-vault
-Disallow: /centos
-Disallow: /ceph
-Disallow: /chakra-releases
-Disallow: /chakra
-Disallow: /chromiumos
-Disallow: /clickhouse
-Disallow: /clojars
-Disallow: /crates.io-index.git
-Disallow: /ctex
-Disallow: /cygwin
-Disallow: /dart-pub
-Disallow: /debian-cd
-Disallow: /debian-multimedia
-Disallow: /debian-nonfree
-Disallow: /debian-security
-Disallow: /debian
-Disallow: /deepin-cd
-Disallow: /deepin
-Disallow: /dell
-Disallow: /docker-ce
-Disallow: /dotdeb
-Disallow: /eclipse
-Disallow: /elasticstack
-Disallow: /elpa
-Disallow: /elrepo
-Disallow: /elvish
-Disallow: /epel
-Disallow: /erlang-solutions
-Disallow: /fdroid
-Disallow: /fedora-altarch
-Disallow: /fedora
-Disallow: /flightgear
-Disallow: /flutter
-Disallow: /fzug
-Disallow: /gcc.git
-Disallow: /gentoo-portage-prefix
-Disallow: /gentoo-portage
-Disallow: /gentoo
-Disallow: /git-repo
-Disallow: /github-release
-Disallow: /gitlab-ce
-Disallow: /gitlab-ci-multi-runner
-Disallow: /gitlab-ee
-Disallow: /gitlab-runner
-Disallow: /glibc.git
-Disallow: /gnu-alpha
-Disallow: /gnu
-Disallow: /grafana
-Disallow: /hackage
-Disallow: /homebrew-bottles
-Disallow: /homebrew
-Disallow: /iina
-Disallow: /influxdata
-Disallow: /ius
-Disallow: /jenkins
-Disallow: /kali-images
-Disallow: /kali
-Disallow: /kernel
-Disallow: /kicad
-Disallow: /kodi
-Disallow: /kubernetes
-Disallow: /libreoffice
-Disallow: /lineage-rom
-Disallow: /lineageOS
-Disallow: /linux-firmware.git
-Disallow: /linux-next.git
-Disallow: /linux-stable.git
-Disallow: /linux.git
-Disallow: /linuxbrew-bottles
-Disallow: /linuxmint-cd
-Disallow: /linuxmint
-Disallow: /llvm
-Disallow: /loongson
-Disallow: /lxc-images
-Disallow: /lyx
-Disallow: /macports
-Disallow: /mageia
-Disallow: /manjaro-arm
-Disallow: /manjaro
-Disallow: /mariadb
-Disallow: /mongodb
-Disallow: /msys2
-Disallow: /mxlinux
-Disallow: /mysql
-Disallow: /neurodebian
-Disallow: /nix-channels
-Disallow: /nix
-Disallow: /nixos-images
-Disallow: /nodejs-release
-Disallow: /nodesource
-Disallow: /openresty
-Disallow: /opensuse
-Disallow: /openthos-src
-Disallow: /openthos
-Disallow: /openwrt
-Disallow: /osdn
-Disallow: /osmc
-Disallow: /packman
-Disallow: /parrot
-Disallow: /percona
-Disallow: /pkgsrc
-Disallow: /postgresql
-Disallow: /puppy
-Disallow: /pybombs
-Disallow: /pypi
-Disallow: /qemu.git
-Disallow: /qt
-Disallow: /qubesos
-Disallow: /raspberrypi
-Disallow: /raspbian-images
-Disallow: /raspbian
-Disallow: /redhat
-Disallow: /remi
-Disallow: /repo-ck
-Disallow: /repoforge
-Disallow: /ros
-Disallow: /rpmfusion
-Disallow: /rubygems
-Disallow: /rustup
-Disallow: /sagemath
-Disallow: /saltstack
-Disallow: /slackware
-Disallow: /slackwarearm
-Disallow: /solus
-Disallow: /stackage
-Disallow: /steamos
-Disallow: /tensorflow
-Disallow: /termux
-Disallow: /tinycorelinux
-Disallow: /ubuntu-cdimage
-Disallow: /ubuntu-cloud-images
-Disallow: /ubuntu-ports
-Disallow: /ubuntu-releases
-Disallow: /ubuntu
-Disallow: /videolan-ftp
-Disallow: /virtualbox
-Disallow: /winehq
-Disallow: /zabbix