auto generate robots.txt

_config.yml

@@ -25,7 +25,7 @@ sass:
 gems:
     - jekyll-babel
 
-exclude: [Gemfile, Gemfile.lock, README.md]
+exclude: [Gemfile, Gemfile.lock, README.md, gen_robot.sh]
 
 # Content Related
 new_mirrors: [openwrt, mongodb, anaconda, packman, gnu, osmc]

gen_robot.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo '# robots.txt for https://mirrors.tuna.tsinghua.edu.cn' > robots.txt
+echo 'User-agent: *' >> robots.txt
+echo '' >> robots.txt
+
+curl -s https://mirrors.tuna.tsinghua.edu.cn/static/tunasync.json | jq -r '.[] | .name' | while read name; do
+	echo "Disallow: /${name}" >> robots.txt
+done

robots.txt

@@ -1,72 +1,66 @@
-# robots.txt for http://mirrors.tuna.tsinghua.edu.cn
-
+# robots.txt for https://mirrors.tuna.tsinghua.edu.cn
 User-agent: *
 
-Disallow: /apache
-Disallow: /android
 Disallow: /AOSP
+Disallow: /CRAN
+Disallow: /CTAN
+Disallow: /OpenBSD
+Disallow: /anaconda
+Disallow: /apache
 Disallow: /archlinux
 Disallow: /archlinuxarm
 Disallow: /archlinuxcn
 Disallow: /centos
 Disallow: /chakra
-Disallow: /CPAN
-Disallow: /CRAN
-Disallow: /CTAN
 Disallow: /cygwin
 Disallow: /debian
-Disallow: /debian-backports
 Disallow: /debian-cd
 Disallow: /debian-multimedia
 Disallow: /debian-security
-Disallow: /debian-weekly-builds
 Disallow: /deepin
-Disallow: /deepin-releases
+Disallow: /deepin-cd
 Disallow: /docker
 Disallow: /dotdeb
-Disallow: /epel
 Disallow: /elrepo
+Disallow: /epel
 Disallow: /fedora
-Disallow: /freebsd
-Disallow: /frugalware
 Disallow: /gentoo
 Disallow: /gentoo-portage
+Disallow: /gitlab-ce
 Disallow: /gnu
 Disallow: /hackage
 Disallow: /homebrew
 Disallow: /ius
 Disallow: /kali
 Disallow: /kali-images
+Disallow: /kali-security
 Disallow: /kernel
+Disallow: /kodi
 Disallow: /linux.git
 Disallow: /linuxmint
 Disallow: /linuxmint-cd
 Disallow: /lxc-images
-Disallow: /macports
+Disallow: /mageia
 Disallow: /manjaro
 Disallow: /mariadb
+Disallow: /mongodb
 Disallow: /msys2
 Disallow: /neurodebian
 Disallow: /nodesource
 Disallow: /opensuse
-Disallow: /OpenBSD
-Disallow: /pypi
+Disallow: /openwrt
+Disallow: /osmc
+Disallow: /packman
 Disallow: /puppy
+Disallow: /pypi
+Disallow: /qt
 Disallow: /raspbian
 Disallow: /repo-ck
 Disallow: /rpmfusion
 Disallow: /sagemath
-Disallow: /scientific
 Disallow: /slackware
-Disallow: /tex-archive
+Disallow: /termux
 Disallow: /ubuntu
-Disallow: /ubuntu-releases
 Disallow: /ubuntu-ports
+Disallow: /ubuntu-releases
 Disallow: /videolan-ftp
-
-Disallow: /stg
-
-Disallow: /git
-Disallow: /gitweb
-
-Disallow: /~