Beacon generates robots.txt per site

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Enhancements
   - Beacon will now automatically generate a `sitemap.xml` for each `beacon_site` defined in the Router
   - Add macro `beacon_sitemap_index` for use in the Router to serve a sitemap index
+  - Beacon will now automatically generate a `robots.txt` for each `beacon_site` defined in the Router
 
 ## 0.3.3 (2024-12-13)
 

lib/beacon/router.ex

@@ -118,7 +118,7 @@ defmodule Beacon.Router do
   @doc """
   Mounts a site in the `prefix` in your host application router.
 
-  This will automatically serve a `sitemap.xml` file from the `prefix` path defined for this site.
+  This will automatically serve `sitemap.xml` and `robots.txt` files from the `prefix` path defined for this site.
 
   ## Options
 
@@ -149,6 +149,7 @@ defmodule Beacon.Router do
           get "/__beacon_assets__/js-:md5", Beacon.Web.AssetsController, :js, assigns: %{site: opts[:site]}
 
           get "/sitemap.xml", Beacon.Web.SitemapController, :show, as: :beacon_sitemap, assigns: %{site: opts[:site]}
+          get "/robots.txt", Beacon.Web.RobotsController, :show, as: :beacon_robots, assigns: %{site: opts[:site]}
 
           # simulate a beacon page inside site prefix so we can check this site is reachable?/2
           get "/__beacon_check__", Beacon.Web.CheckController, :check, metadata: %{site: opts[:site]}

lib/beacon/web/controllers/robots_controller.ex

@@ -0,0 +1,20 @@
+defmodule Beacon.Web.RobotsController do
+  @moduledoc false
+  use Beacon.Web, :controller
+
+  def init(:show), do: :show
+
+  def call(%{assigns: %{site: site}} = conn, :show) do
+    conn
+    |> put_view(Beacon.Web.RobotsTxt)
+    |> put_resp_content_type("text/txt")
+    |> put_resp_header("cache-control", "public max-age=300")
+    |> render(:robots, sitemap_url: get_sitemap_url(conn, site))
+  end
+
+  defp get_sitemap_url(conn, site) do
+    routes_module = Beacon.Loader.fetch_routes_module(site)
+
+    Beacon.apply_mfa(site, routes_module, :beacon_sitemap_url, [conn])
+  end
+end

lib/beacon/web/robots/robots.txt.eex

@@ -0,0 +1,3 @@
+# http://www.robotstxt.org
+User-agent: *
+Sitemap: <%= @sitemap_url %>

lib/beacon/web/robots_txt.ex

@@ -0,0 +1,4 @@
+defmodule Beacon.Web.RobotsTxt do
+  import Phoenix.Template, only: [embed_templates: 1]
+  embed_templates "robots/*.txt"
+end

test/beacon_web/controllers/robots_controller_test.exs

@@ -0,0 +1,26 @@
+defmodule Beacon.Web.RobotsControllerTest do
+  use Beacon.Web.ConnCase, async: false
+
+  test "show", %{conn: conn} do
+    conn = get(conn, "/robots.txt")
+
+    assert response(conn, 200) == """
+           # http://www.robotstxt.org
+           User-agent: *
+           Sitemap: http://localhost:4000/sitemap.xml
+           """
+
+    assert response_content_type(conn, :txt) =~ "charset=utf-8"
+
+    # site: :not_booted
+    conn = get(conn, "/other/robots.txt")
+
+    assert response(conn, 200) == """
+           # http://www.robotstxt.org
+           User-agent: *
+           Sitemap: http://localhost:4000/other/sitemap.xml
+           """
+
+    assert response_content_type(conn, :txt) =~ "charset=utf-8"
+  end
+end