update robots.txt, update config for new mirrors

Signed-off-by: Miao Wang <shankerwangmiao@gmail.com>

_config.yml

@@ -33,13 +33,8 @@ exclude: [Gemfile, Gemfile.lock, README.md, gen_robot.sh, vendor, geninfo/, gen_
 
 # Content Related
 new_mirrors:
-    - fzug
-    - chakra-releases
-    - fedora-altarch
-    - gentoo-portage-prefix
-    - gitlab-runner
-    - pkgsrc
-    - eclipse
-    - nodejs-release
+    - gnu-alpha
+    - mxlinux
+    - mxlinux-isos
 
 # vim: ts=2 sts=2 sw=2 expandtab

robots.txt

@@ -2,6 +2,7 @@
 User-agent: *
 
 Disallow: /AOSP
+Disallow: /CPAN
 Disallow: /CRAN
 Disallow: /CTAN
 Disallow: /CocoaPods
@@ -20,11 +21,15 @@ Disallow: /arch4edu
 Disallow: /archlinux
 Disallow: /archlinuxarm
 Disallow: /archlinuxcn
+Disallow: /armbian-releases
+Disallow: /armbian
+Disallow: /artixlinux
 Disallow: /bananian
 Disallow: /bioconductor
 Disallow: /bjlx
 Disallow: /blackarch
 Disallow: /centos-altarch
+Disallow: /centos-vault
 Disallow: /centos
 Disallow: /ceph
 Disallow: /chakra-releases
@@ -43,13 +48,16 @@ Disallow: /deepin
 Disallow: /dell
 Disallow: /docker-ce
 Disallow: /dotdeb
+Disallow: /eclipse
 Disallow: /elasticstack
 Disallow: /elpa
 Disallow: /elrepo
 Disallow: /elvish
 Disallow: /epel
 Disallow: /erlang-solutions
+Disallow: /fedora-altarch
 Disallow: /fedora
+Disallow: /flightgear
 Disallow: /fzug
 Disallow: /gentoo-portage-prefix
 Disallow: /gentoo-portage
@@ -57,6 +65,9 @@ Disallow: /gentoo
 Disallow: /git-repo
 Disallow: /gitlab-ce
 Disallow: /gitlab-ci-multi-runner
+Disallow: /gitlab-ee
+Disallow: /gitlab-runner
+Disallow: /gnu-alpha
 Disallow: /gnu
 Disallow: /grafana
 Disallow: /hackage
@@ -72,8 +83,10 @@ Disallow: /kali
 Disallow: /kernel
 Disallow: /kodi
 Disallow: /lede
+Disallow: /libreoffice
 Disallow: /lineage-rom
 Disallow: /lineageOS
+Disallow: /linux-next.git
 Disallow: /linux-stable.git
 Disallow: /linux.git
 Disallow: /linuxmint-cd
@@ -88,17 +101,23 @@ Disallow: /manjaro
 Disallow: /mariadb
 Disallow: /mongodb
 Disallow: /msys2
+Disallow: /mxlinux-isos
+Disallow: /mxlinux
 Disallow: /mysql
 Disallow: /neurodebian
+Disallow: /nodejs-release
 Disallow: /nodesource
+Disallow: /openresty
 Disallow: /opensuse
 Disallow: /openthos-src
 Disallow: /openthos
 Disallow: /openwrt
+Disallow: /osdn
 Disallow: /osmc
 Disallow: /packman
 Disallow: /parrot
 Disallow: /percona
+Disallow: /pkgsrc
 Disallow: /postgresql
 Disallow: /puppy
 Disallow: /pybombs