Affichage du schéma d'une ressource si dispo (#1690)

* additionnal fontawesome icon

* add a schema field for the resource

* display schema information if available

* translations

* typo

* open new tab for the schema link

* import resource schema from data.gouv

* add an assert on the schema being imported from data.gouv

* add a field for schema version

* adapt the template to show version if present

and link towards the correct version

* adapt the tests

Co-authored-by: Thibaut Barrère <[email protected]>

apps/db/lib/db/dataset.ex

@@ -97,7 +97,9 @@ defmodule DB.Dataset do
         community_resource_publisher: r.community_resource_publisher,
         original_resource_url: r.original_resource_url,
         features: r.features,
-        modes: r.modes
+        modes: r.modes,
+        schema_name: r.schema_name,
+        schema_version: r.schema_version
       }
     )
   end

apps/db/lib/db/resource.ex

@@ -36,6 +36,10 @@ defmodule DB.Resource do
 
     field(:is_community_resource, :boolean)
 
+    # the declared official schema used by the resource
+    field(:schema_name, :string)
+    field(:schema_version, :string)
+
     # only relevant for community resources, name of the owner or the organization that published the resource
     field(:community_resource_publisher, :string)
     field(:description, :string)
@@ -290,6 +294,8 @@ defmodule DB.Resource do
         :features,
         :modes,
         :is_community_resource,
+        :schema_name,
+        :schema_version,
         :community_resource_publisher,
         :original_resource_url,
         :content_hash,

apps/db/priv/repo/migrations/20210630130031_add_resource_schema.exs

@@ -0,0 +1,10 @@
+defmodule DB.Repo.Migrations.AddResourceSchema do
+  use Ecto.Migration
+
+  def change do
+    alter table(:resource) do
+      add :schema_name, :string
+      add :schema_version, :string
+    end
+  end
+end

apps/transport/client/stylesheets/components/_icons.scss

@@ -81,3 +81,7 @@
 .icon--sync-alt {
   @extend .fa-sync-alt;
 }
+
+.icon--table {
+  @extend .fa-table;
+}

apps/transport/lib/transport/import_data.ex

@@ -317,11 +317,27 @@ defmodule Transport.ImportData do
         "description" => resource["description"],
         "filesize" => resource["filesize"],
         "content_hash" => Hasher.get_content_hash(resource["url"]),
-        "original_resource_url" => get_original_resource_url(resource)
+        "original_resource_url" => get_original_resource_url(resource),
+        "schema_name" => get_schema_name(resource),
+        "schema_version" => get_schema_version(resource)
       }
     end)
   end
 
+  @spec get_schema_name(any) :: binary() | nil
+  def get_schema_name(%{"schema" => %{"name" => schema}}) do
+    schema
+  end
+
+  def get_schema_name(_), do: nil
+
+  @spec get_schema_version(any) :: binary() | nil
+  def get_schema_version(%{"schema" => %{"version" => version}}) do
+    version
+  end
+
+  def get_schema_version(_), do: nil
+
   @spec get_valid_resources(map(), binary()) :: [map()]
   def get_valid_resources(%{"resources" => resources}, "public-transit") do
     resources

apps/transport/lib/transport_web/templates/dataset/_resource.html.eex

@@ -5,6 +5,13 @@
     <%= @resource.title %>
   </h4>
 
+  <%= unless is_nil(@resource.schema_name) do %>
+    <div title="<%= dgettext("page-dataset-details", "Resource declared schema") %>">
+      <i class="icon icon--table" aria-hidden="true"></i>
+      <%= link(schema_label(@resource), to: schema_url(@resource), target: "_blank") %>
+    </div>
+  <% end %>
+
   <%= if Resource.valid_and_available?(@resource) do %>
     <div title="<%= dgettext("page-dataset-details", "Validity period") %>">
       <i class="icon icon--calendar-alt" aria-hidden="true"></i>

apps/transport/lib/transport_web/views/dataset_view.ex

@@ -366,4 +366,18 @@ defmodule TransportWeb.DatasetView do
     |> Enum.sort_by(& &1.metadata["end_date"], &>=/2)
     |> Enum.sort_by(&Resource.valid_and_available?(&1), &>=/2)
   end
+
+  def schema_url(%{schema_name: schema_name, schema_version: schema_version}) when not is_nil(schema_version) do
+    "https://schema.data.gouv.fr/#{schema_name}/#{schema_version}.html"
+  end
+
+  def schema_url(%{schema_name: schema_name}) do
+    "https://schema.data.gouv.fr/#{schema_name}/latest.html"
+  end
+
+  def schema_label(%{schema_name: schema_name, schema_version: schema_version}) when not is_nil(schema_version) do
+    "#{schema_name} (#{schema_version})"
+  end
+
+  def schema_label(%{schema_name: schema_name}), do: schema_name
 end

apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po

@@ -377,3 +377,7 @@ msgstr ""
 #, elixir-format
 msgid "Data producer"
 msgstr ""
+
+#, elixir-format
+msgid "Resource declared schema"
+msgstr ""

apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po

@@ -377,3 +377,7 @@ msgstr "Les réutilisations sont temporairement indisponibles"
 #, elixir-format
 msgid "Data producer"
 msgstr "Producteur de la donnée"
+
+#, elixir-format
+msgid "Resource declared schema"
+msgstr "La ressource déclare utiliser ce schéma"

apps/transport/priv/gettext/page-dataset-details.pot

@@ -377,3 +377,7 @@ msgstr ""
 #, elixir-format
 msgid "Data producer"
 msgstr ""
+
+#, elixir-format
+msgid "Resource declared schema"
+msgstr ""

apps/transport/test/transport/import_data_test.exs

@@ -17,12 +17,13 @@ defmodule Transport.ImportDataTest do
     :ok
   end
 
-  def generate_resources_payload(title \\ nil, url \\ nil, id \\ nil) do
+  def generate_resources_payload(title \\ nil, url \\ nil, id \\ nil, schema_name \\ nil, schema_version \\ nil) do
     [
       %{
         "title" => title || "resource1",
         "url" => url || "http://localhost:4321/resource1",
-        "id" => id || "resource1_id"
+        "id" => id || "resource1_id",
+        "schema" => %{"name" => schema_name, "version" => schema_version}
       }
     ]
   end
@@ -226,12 +227,16 @@ defmodule Transport.ImportDataTest do
     assert db_count(DB.Resource) == 0
 
     community_resource_title = "a_community_resource"
+    schema_name = "etalab/covoiturage"
+    schema_version = "v1.0"
 
     with_mock HTTPoison,
       get!: http_get_mock_200(datagouv_id, generate_dataset_payload(datagouv_id, [])),
       head: http_head_mock() do
       with_mock Datagouvfr.Client.CommunityResources,
-        get: fn _ -> {:ok, generate_resources_payload(community_resource_title)} end do
+        get: fn _ ->
+          {:ok, generate_resources_payload(community_resource_title, "url", "1", schema_name, schema_version)}
+        end do
         with_mock HTTPStreamV2, fetch_status_and_hash: http_stream_mock() do
           ImportData.import_all_datasets()
         end
@@ -240,6 +245,8 @@ defmodule Transport.ImportDataTest do
 
     [resource] = DB.Resource |> where([r], r.is_community_resource) |> DB.Repo.all()
     assert Map.get(resource, :title) == community_resource_title
+    assert Map.get(resource, :schema_name) == schema_name
+    assert Map.get(resource, :schema_version) == schema_version
   end
 
   # test "error while connecting to datagouv server"