diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc5d6e6..b7b6da17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements - Beacon will now automatically generate a `sitemap.xml` for each `beacon_site` defined in the Router - Add macro `beacon_sitemap_index` for use in the Router to serve a sitemap index + - Beacon will now automatically generate a `robots.txt` for each `beacon_site` defined in the Router ## 0.3.3 (2024-12-13) diff --git a/lib/beacon/router.ex b/lib/beacon/router.ex index ffcbed91..7ebf3247 100644 --- a/lib/beacon/router.ex +++ b/lib/beacon/router.ex @@ -118,7 +118,7 @@ defmodule Beacon.Router do @doc """ Mounts a site in the `prefix` in your host application router. - This will automatically serve a `sitemap.xml` file from the `prefix` path defined for this site. + This will automatically serve `sitemap.xml` and `robots.txt` files from the `prefix` path defined for this site. ## Options @@ -149,6 +149,7 @@ defmodule Beacon.Router do get "/__beacon_assets__/js-:md5", Beacon.Web.AssetsController, :js, assigns: %{site: opts[:site]} get "/sitemap.xml", Beacon.Web.SitemapController, :show, as: :beacon_sitemap, assigns: %{site: opts[:site]} + get "/robots.txt", Beacon.Web.RobotsController, :show, as: :beacon_robots, assigns: %{site: opts[:site]} # simulate a beacon page inside site prefix so we can check this site is reachable?/2 get "/__beacon_check__", Beacon.Web.CheckController, :check, metadata: %{site: opts[:site]} diff --git a/lib/beacon/web/controllers/robots_controller.ex b/lib/beacon/web/controllers/robots_controller.ex new file mode 100644 index 00000000..fd906a90 --- /dev/null +++ b/lib/beacon/web/controllers/robots_controller.ex @@ -0,0 +1,20 @@ +defmodule Beacon.Web.RobotsController do + @moduledoc false + use Beacon.Web, :controller + + def init(:show), do: :show + + def call(%{assigns: %{site: site}} = conn, :show) do + conn + |> put_view(Beacon.Web.RobotsTxt) + |> put_resp_content_type("text/txt") + |> put_resp_header("cache-control", "public max-age=300") + |> render(:robots, sitemap_url: get_sitemap_url(conn, site)) + end + + defp get_sitemap_url(conn, site) do + routes_module = Beacon.Loader.fetch_routes_module(site) + + Beacon.apply_mfa(site, routes_module, :beacon_sitemap_url, [conn]) + end +end diff --git a/lib/beacon/web/robots/robots.txt.eex b/lib/beacon/web/robots/robots.txt.eex new file mode 100644 index 00000000..6a683c9f --- /dev/null +++ b/lib/beacon/web/robots/robots.txt.eex @@ -0,0 +1,3 @@ +# http://www.robotstxt.org +User-agent: * +Sitemap: <%= @sitemap_url %> diff --git a/lib/beacon/web/robots_txt.ex b/lib/beacon/web/robots_txt.ex new file mode 100644 index 00000000..22fc32a4 --- /dev/null +++ b/lib/beacon/web/robots_txt.ex @@ -0,0 +1,4 @@ +defmodule Beacon.Web.RobotsTxt do + import Phoenix.Template, only: [embed_templates: 1] + embed_templates "robots/*.txt" +end diff --git a/test/beacon_web/controllers/robots_controller_test.exs b/test/beacon_web/controllers/robots_controller_test.exs new file mode 100644 index 00000000..570a194d --- /dev/null +++ b/test/beacon_web/controllers/robots_controller_test.exs @@ -0,0 +1,26 @@ +defmodule Beacon.Web.RobotsControllerTest do + use Beacon.Web.ConnCase, async: false + + test "show", %{conn: conn} do + conn = get(conn, "/robots.txt") + + assert response(conn, 200) == """ + # http://www.robotstxt.org + User-agent: * + Sitemap: http://localhost:4000/sitemap.xml + """ + + assert response_content_type(conn, :txt) =~ "charset=utf-8" + + # site: :not_booted + conn = get(conn, "/other/robots.txt") + + assert response(conn, 200) == """ + # http://www.robotstxt.org + User-agent: * + Sitemap: http://localhost:4000/other/sitemap.xml + """ + + assert response_content_type(conn, :txt) =~ "charset=utf-8" + end +end