docs: add /robots.txt (#1143)

golangci · May 18, 2020 · 2c45244 · 2c45244
1 parent f0c0f06
commit 2c45244
Show file tree

Hide file tree

Showing 3 changed files with 229 additions and 1 deletion.
diff --git a/docs/gatsby-config.js b/docs/gatsby-config.js
@@ -1,6 +1,7 @@
 const withDefault = require(`./src/@rocketseat/gatsby-theme-docs-core/util/with-default`);
 
-const siteUrl = `https://golangci-lint.run`;
+const domain = `golangci-lint.run`;
+const siteUrl = `https://${domain}`;
 
 const siteConfig = require(`./src/config/site.js`);
 const { basePath, configPath, docsPath } = withDefault(siteConfig);
@@ -112,5 +113,20 @@ module.exports = {
     `gatsby-plugin-catch-links`,
     `gatsby-plugin-emotion`,
     `gatsby-plugin-react-helmet`,
+    {
+      resolve: `gatsby-plugin-robots-txt`,
+      options: {
+        env: {
+          development: {
+            host: domain,
+            policy: [{ userAgent: "*", disallow: ["/"] }],
+          },
+          production: {
+            host: domain,
+            policy: [{ userAgent: "*", disallow: ["/page-data/"] }],
+          },
+        },
+      },
+    },
   ],
 };
diff --git a/docs/package-lock.json b/docs/package-lock.json
diff --git a/docs/package.json b/docs/package.json
@@ -24,6 +24,7 @@
     "gatsby-plugin-offline": "^3.0.32",
     "gatsby-plugin-react-helmet": "^3.1.16",
     "gatsby-plugin-react-svg": "^3.0.0",
+    "gatsby-plugin-robots-txt": "^1.5.0",
     "gatsby-plugin-sharp": "^2.4.0",
     "gatsby-plugin-sitemap": "^2.2.26",
     "gatsby-remark-autolink-headers": "^2.1.23",