add robots.txt

Signed-off-by: liuboaibc <kukudehero@gmail.com>

config/_default/config.toml

@@ -1,5 +1,7 @@
 baseURL = "https://kubesphere-v3.netlify.app"
 
+enableRobotsTXT = true
+
 [markup]
   [markup.tableOfContents]
     endLevel = 3

content/en/api/crd.md

@@ -2,7 +2,7 @@
 title: KubeSphere Api Documents
 description: KubeSphere Api Documents
 keywords: KubeSphere, KubeSphere Documents, Kubernetes
-
+disallow: true
 swaggerUrl: json/crd.json
 ---
 

content/en/api/kubesphere.md

@@ -2,6 +2,6 @@
 title: KubeSphere Api Documents
 description: KubeSphere Api Documents
 keywords: KubeSphere, KubeSphere Documents, Kubernetes
-
+disallow: true
 swaggerUrl: json/kubesphere.json
 ---

content/en/contribution/request.md

@@ -1,7 +1,7 @@
 ---
 title: "contribution request"
 layout: "request"
-
+disallow: true
 css: "scss/contribution.scss"
 
 section1:

content/en/partner/request.md

@@ -2,7 +2,7 @@
 title: "partner request"
 
 layout: "request"
-
+disallow: true
 css: "scss/partner.scss"
 
 section1:

content/zh/contribution/request.md

@@ -1,7 +1,7 @@
 ---
 title: "contribution request"
 layout: "request"
-
+disallow: true
 css: "scss/contribution.scss"
 
 section1:

content/zh/partner/request.md

@@ -2,7 +2,7 @@
 title: "partner request"
 
 layout: "request"
-
+disallow: true
 css: "scss/partner.scss"
 
 section1:

layouts/robots.txt

@@ -0,0 +1,6 @@
+User-agent: *
+{{ range .Pages }}
+{{ if .Params.disallow }}
+Disallow: {{ .RelPermalink }}
+{{ end }}
+{{ end }}