Merge branch 'Simon-Initiative:master' into master

config/config.exs

@@ -154,7 +154,8 @@ config :oli, Oban,
     grades: 30,
     auto_submit: 3,
     analytics_export: 3,
-    datashop_export: 3
+    datashop_export: 3,
+    objectives: 3
   ]
 
 config :ex_money,

lib/mix/tasks/create_contained_objectives.ex

@@ -0,0 +1,52 @@
+defmodule Mix.Tasks.CreateContainedObjectives do
+  @shortdoc "Create contained objectives for all sections that were not migrated"
+
+  use Mix.Task
+
+  alias Oli.Delivery.Sections
+  alias Oli.Repo
+  alias Oli.Delivery.Sections.{ContainedObjectivesBuilder, Section}
+  alias Ecto.Multi
+
+  require Logger
+
+  import Ecto.Query, only: [from: 2]
+
+  @impl Mix.Task
+  def run(_args) do
+    Mix.Task.run("app.start")
+
+    run_now()
+  end
+
+  def run_now() do
+    Logger.info("Start enqueueing contained objectives creation")
+
+    Multi.new()
+    |> Multi.run(:sections, &get_not_started_sections(&1, &2))
+    |> Oban.insert_all(:jobs, &build_contained_objectives_jobs(&1))
+    |> Ecto.Multi.update_all(
+      :update_all_sections,
+      fn _ -> from(Section, where: [v25_migration: :not_started]) end,
+      set: [v25_migration: :pending]
+    )
+    |> Repo.transaction()
+    |> case do
+      {:ok, %{jobs: jobs}} ->
+        Logger.info("#{Enum.count(jobs)} jobs enqueued for contained objectives creation")
+
+        :ok
+
+      {:error, _, changeset, _} ->
+        Logger.error("Error enqueuing jobs: #{inspect(changeset)}")
+
+        :error
+    end
+  end
+
+  defp get_not_started_sections(_repo, _changes),
+    do: {:ok, Sections.get_sections_by([v25_migration: :not_started], [:slug])}
+
+  defp build_contained_objectives_jobs(%{sections: sections}),
+    do: Enum.map(sections, &ContainedObjectivesBuilder.new(%{section_slug: &1.slug}))
+end

lib/oli/delivery.ex

@@ -105,6 +105,7 @@ defmodule Oli.Delivery do
         )
 
       {:ok, _} = Sections.rebuild_contained_pages(section)
+      {:ok, _} = Sections.rebuild_contained_objectives(section)
 
       enroll(user.id, section.id, lti_params)
 
@@ -153,6 +154,7 @@ defmodule Oli.Delivery do
 
       {:ok, %Section{} = section} = Sections.create_section_resources(section, publication)
       {:ok, _} = Sections.rebuild_contained_pages(section)
+      {:ok, _} = Sections.rebuild_contained_objectives(section)
 
       enroll(user.id, section.id, lti_params)
       {:ok, updated_section} = maybe_update_section_contains_explorations(section)

lib/oli/delivery/sections.ex

@@ -7,9 +7,7 @@ defmodule Oli.Delivery.Sections do
   alias Oli.Delivery.Sections.EnrollmentContextRole
   alias Oli.Repo
   alias Oli.Repo.{Paging, Sorting}
-  alias Oli.Delivery.Sections.Section
-  alias Oli.Delivery.Sections.ContainedPage
-  alias Oli.Delivery.Sections.Enrollment
+  alias Oli.Delivery.Sections.{ContainedObjective, ContainedPage, Enrollment, Section}
   alias Lti_1p3.Tool.ContextRole
   alias Lti_1p3.DataProviders.EctoProvider
   alias Oli.Lti.Tool.{Deployment, Registration}
@@ -1857,6 +1855,7 @@ defmodule Oli.Delivery.Sections do
       PreviousNextIndex.rebuild(section)
 
       {:ok, _} = rebuild_contained_pages(section, section_resources)
+      {:ok, _} = rebuild_contained_objectives(section)
 
       section_resources
     end)
@@ -1999,6 +1998,138 @@ defmodule Oli.Delivery.Sections do
     Ecto.Adapters.SQL.query(Repo, sql, [section_id, section_id])
   end
 
+  @doc """
+  Rebuilds the "contained objectives" relations for a course section. A "contained objective" for a
+  container is the full set of objectives found within the activities (within the pages) included in the container or in any of
+  its sub-containers.  For every container in a course section, one row will exist in this
+  "contained objectives" table for each contained objective. This allows a straightforward join through
+  this relation from a container to then all of its contained objectives.
+  It does not take into account the objectives attached to the pages within a container.
+
+  There will be always at least one entry per objective with the container_id being nil, which represents the inclusion of the objective in the root container.
+  """
+
+  def rebuild_contained_objectives(section) do
+    timestamps = %{
+      inserted_at: {:placeholder, :now},
+      updated_at: {:placeholder, :now}
+    }
+
+    placeholders = %{
+      now: DateTime.utc_now() |> DateTime.truncate(:second)
+    }
+
+    Multi.new()
+    |> Multi.delete_all(
+      :delete_all_objectives,
+      from(ContainedObjective, where: [section_id: ^section.id])
+    )
+    |> Multi.run(:contained_objectives, &build_contained_objectives(&1, &2, section.slug))
+    |> Multi.insert_all(
+      :inserted_contained_objectives,
+      ContainedObjective,
+      &objectives_with_timestamps(&1, timestamps),
+      placeholders: placeholders
+    )
+    |> Repo.transaction()
+    |> case do
+      {:ok, res} ->
+        {:ok, res}
+
+      {:error, _, changeset, _} ->
+        {:error, changeset}
+    end
+  end
+
+  def build_contained_objectives(repo, _changes, section_slug) do
+    page_type_id = ResourceType.get_id_by_type("page")
+    activity_type_id = ResourceType.get_id_by_type("activity")
+
+    section_resource_pages =
+      from(
+        [sr: sr, rev: rev, s: s] in DeliveryResolver.section_resource_revisions(section_slug),
+        where: not rev.deleted and rev.resource_type_id == ^page_type_id
+      )
+
+    section_resource_activities =
+      from(
+        [sr: sr, rev: rev, s: s] in DeliveryResolver.section_resource_revisions(section_slug),
+        where: not rev.deleted and rev.resource_type_id == ^activity_type_id,
+        select: rev
+      )
+
+    activity_references =
+      from(
+        rev in Revision,
+        join: content_elem in fragment("jsonb_array_elements(?->'model')", rev.content),
+        select: %{
+          revision_id: rev.id,
+          activity_id: fragment("(?->>'activity_id')::integer", content_elem)
+        },
+        where: fragment("?->>'type'", content_elem) == "activity-reference"
+      )
+
+    activity_objectives =
+      from(
+        rev in Revision,
+        join: obj in fragment("jsonb_each_text(?)", rev.objectives),
+        select: %{
+          objective_revision_id: rev.id,
+          objective_resource_id:
+            fragment("jsonb_array_elements_text(?::jsonb)::integer", obj.value)
+        },
+        where: rev.deleted == false and rev.resource_type_id == ^activity_type_id
+      )
+
+    contained_objectives =
+      from(
+        [sr: sr, rev: rev, s: s] in section_resource_pages,
+        join: cp in ContainedPage,
+        on: cp.page_id == rev.resource_id and cp.section_id == s.id,
+        join: ar in subquery(activity_references),
+        on: ar.revision_id == rev.id,
+        join: act in subquery(section_resource_activities),
+        on: act.resource_id == ar.activity_id,
+        join: ao in subquery(activity_objectives),
+        on: ao.objective_revision_id == act.id,
+        group_by: [cp.section_id, cp.container_id, ao.objective_resource_id],
+        select: %{
+          section_id: cp.section_id,
+          container_id: cp.container_id,
+          objective_id: ao.objective_resource_id
+        }
+      )
+      |> repo.all()
+
+    {:ok, contained_objectives}
+  end
+
+  defp objectives_with_timestamps(%{contained_objectives: contained_objectives}, timestamps) do
+    Enum.map(contained_objectives, &Map.merge(&1, timestamps))
+  end
+
+  @doc """
+  Returns the contained objectives for a given section and container.
+  If the container id is nil, then it returns the contained objectives for the root container (all objectives of the section).
+  """
+  def get_section_contained_objectives(section_id, nil) do
+    Repo.all(
+      from(co in ContainedObjective,
+        where: co.section_id == ^section_id and is_nil(co.container_id),
+        select: co.objective_id
+      )
+    )
+  end
+
+  def get_section_contained_objectives(section_id, container_id) do
+    Repo.all(
+      from(co in ContainedObjective,
+        where: [section_id: ^section_id, container_id: ^container_id],
+        select: co.objective_id
+      )
+    )
+  end
+
   @doc """
   Gracefully applies the specified publication update to a given section by leaving the existing
   curriculum and section modifications in-tact while applying the structural changes that
@@ -3364,4 +3495,21 @@ defmodule Oli.Delivery.Sections do
       )
     )
   end
+
+  @doc """
+    Get all sections filtered by the clauses passed as the first argument.
+    The second argument is a list of fields to be selected from the Section table.
+    If the second argument is not passed, all fields will be selected.
+  """
+  def get_sections_by(clauses, select_fields \\ nil) do
+    Section
+    |> from(where: ^clauses)
+    |> maybe_select_section_fields(select_fields)
+    |> Repo.all()
+  end
+
+  defp maybe_select_section_fields(query, nil), do: query
+
+  defp maybe_select_section_fields(query, select_fields),
+    do: select(query, [s], struct(s, ^select_fields))
 end

lib/oli/delivery/sections/contained_objective.ex

@@ -0,0 +1,39 @@
+defmodule Oli.Delivery.Sections.ContainedObjective do
+  @moduledoc """
+  The ContainedObjective schema represents an association between section containers and objectives linked to the pages within them.
+  It is created for optimization purposes since it provides a fast way of retrieving the objectives associated to the containers of a section.
+
+  Given a section, each objective will have as many entries in the table as containers it is linked to.
+  There will be always at least one entry per objective with the container_id being nil, which represents the inclusion of the objective in the root container.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  alias Oli.Delivery.Sections.Section
+  alias Oli.Resources.Resource
+
+  schema "contained_objectives" do
+    belongs_to(:section, Section)
+    belongs_to(:container, Resource)
+    belongs_to(:objective, Resource)
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(contained_page, attrs) do
+    contained_page
+    |> cast(attrs, [
+      :section_id,
+      :container_id,
+      :objective_id
+    ])
+    |> validate_required([
+      :section_id,
+      :objective_id
+    ])
+    |> foreign_key_constraint(:section_id)
+    |> foreign_key_constraint(:objective_id)
+  end
+end

lib/oli/delivery/sections/contained_objectives_builder.ex

@@ -0,0 +1,62 @@
+defmodule Oli.Delivery.Sections.ContainedObjectivesBuilder do
+  use Oban.Worker,
+    queue: :objectives,
+    unique: [keys: [:section_slug]],
+    max_attempts: 1
+
+  alias Oli.Delivery.Sections.{ContainedObjective, Section}
+  alias Oli.Delivery.Sections
+  alias Oli.Repo
+  alias Ecto.Multi
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"section_slug" => section_slug}}) do
+    timestamps = %{
+      inserted_at: {:placeholder, :now},
+      updated_at: {:placeholder, :now}
+    }
+
+    placeholders = %{
+      now: DateTime.utc_now() |> DateTime.truncate(:second)
+    }
+
+    Multi.new()
+    |> Multi.run(
+      :contained_objectives,
+      &Sections.build_contained_objectives(&1, &2, section_slug)
+    )
+    |> Multi.insert_all(
+      :inserted_contained_objectives,
+      ContainedObjective,
+      &objectives_with_timestamps(&1, timestamps),
+      placeholders: placeholders
+    )
+    |> Multi.run(:section, &find_section_by_slug(&1, &2, section_slug))
+    |> Multi.update(
+      :done_section,
+      &Section.changeset(&1.section, %{v25_migration: :done})
+    )
+    |> Repo.transaction()
+    |> case do
+      {:ok, res} ->
+        {:ok, res}
+
+      {:error, _, changeset, _} ->
+        {:error, changeset}
+    end
+  end
+
+  defp objectives_with_timestamps(%{contained_objectives: contained_objectives}, timestamps) do
+    Enum.map(contained_objectives, &Map.merge(&1, timestamps))
+  end
+
+  defp find_section_by_slug(repo, _changes, section_slug) do
+    case repo.get_by(Section, slug: section_slug) do
+      nil ->
+        {:error, :section_not_found}
+
+      section ->
+        {:ok, section}
+    end
+  end
+end

lib/oli/delivery/sections/section.ex

@@ -127,6 +127,11 @@ defmodule Oli.Delivery.Sections.Section do
 
     field(:preferred_scheduling_time, :time, default: ~T[23:59:59])
 
+    field(:v25_migration, Ecto.Enum,
+      values: [:not_started, :done, :pending],
+      default: :done
+    )
+
     timestamps(type: :utc_datetime)
   end
 
@@ -179,7 +184,8 @@ defmodule Oli.Delivery.Sections.Section do
       :class_modality,
       :class_days,
       :course_section_number,
-      :preferred_scheduling_time
+      :preferred_scheduling_time,
+      :v25_migration
     ])
     |> cast_embed(:customizations, required: false)
     |> validate_required([

lib/oli_web/controllers/open_and_free_controller.ex

@@ -338,6 +338,7 @@ defmodule OliWeb.OpenAndFreeController do
     Repo.transaction(fn ->
       with {:ok, section} <- Oli.Delivery.Sections.Blueprint.duplicate(blueprint, section_params),
            {:ok, _} <- Sections.rebuild_contained_pages(section),
+           {:ok, _} <- Sections.rebuild_contained_objectives(section),
            {:ok, _maybe_enrollment} <- enroll(conn, section) do
         section
       else
@@ -351,6 +352,7 @@ defmodule OliWeb.OpenAndFreeController do
       with {:ok, section} <- Sections.create_section(section_params),
            {:ok, section} <- Sections.create_section_resources(section, publication),
            {:ok, _} <- Sections.rebuild_contained_pages(section),
+           {:ok, _} <- Sections.rebuild_contained_objectives(section),
            {:ok, _enrollment} <- enroll(conn, section),
            {:ok, updated_section} <- Delivery.maybe_update_section_contains_explorations(section) do
         updated_section