docs: generate docs sitemap and robots.txt (#6855)

nopcoder · web-flow · commit 804c5ee3c142 · 2023-10-25T14:20:52.000+03:00
* docs: add sitemap to site

* remove asserts from sitemap

* fix asserts exclude and add user-agent to robots

* add missing file

* Add jekyll seo plugin to Gemfile

* fix conflict

* update Gemfile.lock
diff --git a/docs/Gemfile b/docs/Gemfile
@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 gem 'github-pages', group: :jekyll_plugins
 gem 'just-the-docs'
 gem "webrick", "~> 1.8"
+gem 'jekyll-sitemap'
+gem 'jekyll-seo-tag'
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
@@ -261,6 +261,8 @@ PLATFORMS
 
 DEPENDENCIES
   github-pages
+  jekyll-seo-tag
+  jekyll-sitemap
   just-the-docs
   webrick (~> 1.8)
 
diff --git a/docs/_config.yml b/docs/_config.yml
@@ -27,6 +27,10 @@ defaults:
       path: ""
     values:
       image: https://docs.lakefs.io/assets/img/docs_logo.png
+  - scope:
+      path: "assets"
+    values:
+      sitemap: false
 
 # general config
 repository: treeverse/lakeFS
@@ -36,6 +40,7 @@ remote_theme: pmarsceill/just-the-docs@v0.6.0
 plugins:
   - jekyll-redirect-from
   - jekyll-seo-tag
+  - jekyll-sitemap
 
 exclude: 
   - howto/deploy/includes
@@ -120,4 +125,4 @@ callouts:
   tip:
     color: green
   fubar:
-    color: purple
+    color: purple
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1,6 @@
+---
+layout: null
+sitemap: false
+---
+User-agent: *
+Sitemap: {{ site.url }}/sitemap.xml
