From b89bb6c863893ccdd376bcb069e6e93044aac1e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=A1s=20Cirio?= <nicolas.cirio@wyeworks.com>
Date: Tue, 21 May 2024 14:47:30 -0300
Subject: [PATCH 1/2] [FEATURE] [NG23-156] Create in context learning
 proficiency (#4821)

* add proficiency icons

* add proficiency_for_student_per_learning_objective/2 to Oli.Delivery.Metrics

* list learning objectives with proficiency on page header

* proficiency icon with tooltip

* implement light mode

* test new proficiency_for_student_per_learning_objective/2

* add more tests

* assign student objectives metrics for review live

* proficiency explanation modal in dark mode

* proficiency explanation modal in light mode

* add test

---------

Co-authored-by: Darren Siegel <siegel.darren@gmail.com>
---
 lib/oli/delivery/metrics.ex                   |  71 ++++++++++
 lib/oli/resources.ex                          |  11 ++
 lib/oli_web/components/modal.ex               | 122 +++++++++++++++-
 lib/oli_web/icons.ex                          | 101 ++++++++++++++
 .../live/delivery/student/lesson_live.ex      |  55 +++++++-
 .../live/delivery/student/review_live.ex      |  38 +++++
 lib/oli_web/live/delivery/student/utils.ex    | 130 +++++++++++++++---
 .../oli/analytics/summary/metrics_v2_test.exs |  48 ++++---
 .../metrics/learning_proficiency_test.exs     |  22 +++
 .../delivery/student/lesson_live_test.exs     | 121 +++++++++++++++-
 test/support/test_helpers.ex                  |  18 +++
 11 files changed, 694 insertions(+), 43 deletions(-)

diff --git a/lib/oli/delivery/metrics.ex b/lib/oli/delivery/metrics.ex
index 1901e70a7bb..6b42c8855d0 100644
--- a/lib/oli/delivery/metrics.ex
+++ b/lib/oli/delivery/metrics.ex
@@ -1158,6 +1158,77 @@ defmodule Oli.Delivery.Metrics do
     end)
   end
 
+  @doc """
+  Calculates the learning proficiency ("High", "Medium", "Low", "Not enough data")
+  for a given student in a given section.
+
+      It returns a map:
+
+      %{objective_id_1 => "High",
+        ...
+        objective_id_n => "Low"
+      }
+  """
+  @spec proficiency_for_student_per_learning_objective(
+          section :: %Oli.Delivery.Sections.Section{},
+          student_id :: integer
+        ) :: map
+  def proficiency_for_student_per_learning_objective(
+        %Section{slug: section_slug, analytics_version: :v1},
+        student_id
+      ) do
+    query =
+      from(sn in Snapshot,
+        join: s in Section,
+        on: sn.section_id == s.id,
+        where:
+          sn.attempt_number == 1 and sn.part_attempt_number == 1 and s.slug == ^section_slug and
+            sn.user_id == ^student_id,
+        group_by: sn.objective_id,
+        select:
+          {sn.objective_id,
+           fragment(
+             "CAST(COUNT(CASE WHEN ? THEN 1 END) as float) / CAST(COUNT(*) as float)",
+             sn.correct
+           )}
+      )
+
+    Repo.all(query)
+    |> Enum.into(%{}, fn {objective_id, proficiency} ->
+      {objective_id, proficiency_range(proficiency)}
+    end)
+  end
+
+  def proficiency_for_student_per_learning_objective(
+        %Section{id: section_id, analytics_version: :v2},
+        student_id
+      ) do
+    objective_type_id = Oli.Resources.ResourceType.id_for_objective()
+
+    query =
+      from(summary in Oli.Analytics.Summary.ResourceSummary,
+        where:
+          summary.section_id == ^section_id and
+            summary.project_id == -1 and
+            summary.publication_id == -1 and
+            summary.user_id == ^student_id and
+            summary.resource_type_id == ^objective_type_id,
+        group_by: summary.resource_id,
+        select:
+          {summary.resource_id,
+           fragment(
+             "CAST(SUM(?) as float) / CAST(SUM(?) as float)",
+             summary.num_first_attempts_correct,
+             summary.num_first_attempts
+           )}
+      )
+
+    Repo.all(query)
+    |> Enum.into(%{}, fn {objective_id, proficiency} ->
+      {objective_id, proficiency_range(proficiency)}
+    end)
+  end
+
   def proficiency_range(nil), do: "Not enough data"
   def proficiency_range(proficiency) when proficiency <= 0.5, do: "Low"
   def proficiency_range(proficiency) when proficiency <= 0.8, do: "Medium"
diff --git a/lib/oli/resources.ex b/lib/oli/resources.ex
index 1bb3d1db484..9557a652ef0 100644
--- a/lib/oli/resources.ex
+++ b/lib/oli/resources.ex
@@ -425,4 +425,15 @@ defmodule Oli.Resources do
     )
     |> Repo.one()
   end
+
+  @doc """
+  Returns a list of revisions for the given resource ids.
+  """
+  def get_revisions_by_resource_id(resource_ids) do
+    from(r in Revision,
+      where: r.resource_id in ^resource_ids,
+      select: r
+    )
+    |> Repo.all()
+  end
 end
diff --git a/lib/oli_web/components/modal.ex b/lib/oli_web/components/modal.ex
index b6e50f6e437..b79da063ded 100644
--- a/lib/oli_web/components/modal.ex
+++ b/lib/oli_web/components/modal.ex
@@ -68,7 +68,7 @@ defmodule OliWeb.Components.Modal do
               phx-window-keydown={hide_modal(@on_cancel, @id)}
               phx-key="escape"
               phx-click-away={hide_modal(@on_cancel, @id)}
-              class="hidden relative rounded-lg bg-white dark:bg-body-dark shadow-lg shadow-zinc-700/10 ring-1 ring-zinc-700/10 transition"
+              class="hidden relative bg-white dark:bg-body-dark shadow-lg shadow-zinc-700/10 ring-1 ring-zinc-700/10 transition"
             >
               <!-- Modal header -->
               <div class="flex items-start justify-between p-4">
@@ -148,6 +148,126 @@ defmodule OliWeb.Components.Modal do
     """
   end
 
+  attr :id, :string, required: true
+  attr :class, :string, default: ""
+  attr :body_class, :string, default: "p-6 space-y-6"
+  attr :show, :boolean, default: false
+  attr :on_cancel, JS, default: %JS{}
+  attr :on_confirm, JS, default: %JS{}
+
+  slot :inner_block, required: true
+  slot :title
+  slot :subtitle
+  slot :confirm
+  slot :cancel
+
+  def student_delivery_modal(assigns) do
+    ~H"""
+    <div
+      id={@id}
+      phx-mounted={@show && show_modal(@id)}
+      phx-remove={hide_modal(@id)}
+      class="relative z-50 hidden"
+    >
+      <div
+        id={"#{@id}-bg"}
+        class="fixed inset-0 bg-black/20 transition-opacity backdrop-blur-sm"
+        aria-hidden="true"
+      />
+      <div
+        class="fixed inset-0 overflow-y-auto"
+        aria-labelledby={"#{@id}-title"}
+        aria-describedby={"#{@id}-description"}
+        role="dialog"
+        aria-modal="true"
+        tabindex="0"
+      >
+        <div class="flex min-h-full items-center justify-center">
+          <div class={["w-full p-4 sm:p-6 lg:py-8", @class]}>
+            <.focus_wrap
+              id={"#{@id}-container"}
+              phx-mounted={@show && show_modal(@id)}
+              phx-window-keydown={hide_modal(@on_cancel, @id)}
+              phx-key="escape"
+              phx-click-away={hide_modal(@on_cancel, @id)}
+              class="hidden p-8 sm:p-16 lg:p-20 xl:p-28 relative bg-white dark:bg-black shadow-lg shadow-zinc-700/10 ring-1 ring-zinc-700/10 transition"
+            >
+              <button
+                type="button"
+                class="absolute top-5 right-5 lg:top-10 lg:right-10 dark:text-gray-400 dark:hover:text-white text-gray-900 hover:text-gray-400 text-sm w-8 h-8  flex items-center justify-center"
+                phx-click={hide_modal(@on_cancel, @id)}
+                aria-label={gettext("close")}
+              >
+                <svg
+                  class="w-3 h-3"
+                  aria-hidden="true"
+                  xmlns="http://www.w3.org/2000/svg"
+                  fill="none"
+                  viewBox="0 0 14 14"
+                >
+                  <path
+                    stroke="currentColor"
+                    stroke-linecap="round"
+                    stroke-linejoin="round"
+                    stroke-width="2"
+                    d="m1 1 6 6m0 0 6 6M7 7l6-6M7 7l-6 6"
+                  />
+                </svg>
+                <span class="sr-only">Close modal</span>
+              </button>
+              <!-- Modal header -->
+              <div class="flex items-start justify-between">
+                <div :if={@title != []} class="mb-11">
+                  <h1
+                    id={"#{@id}-title"}
+                    class="text-zinc-700 dark:text-neutral-300 text-[40px] font-bold font-['Inter'] leading-[60px]"
+                  >
+                    <%= render_slot(@title) %>
+                  </h1>
+                  <p
+                    :if={@subtitle != []}
+                    id={"#{@id}-description"}
+                    class="mt-11 text-zinc-700 dark:text-white text-base font-normal font-['Inter'] leading-normal"
+                  >
+                    <%= render_slot(@subtitle) %>
+                  </p>
+                </div>
+              </div>
+              <!-- Modal body -->
+              <div class={@body_class}>
+                <%= render_slot(@inner_block) %>
+              </div>
+              <!-- Modal footer -->
+              <div :if={@confirm != [] or @cancel != []}>
+                <div class="flex justify-end p-6 space-x-2">
+                  <.button
+                    :for={cancel <- @cancel}
+                    phx-click={hide_modal(@on_cancel, @id)}
+                    class="bg-transparent text-blue-500 hover:underline hover:bg-transparent"
+                  >
+                    <%= render_slot(cancel) %>
+                  </.button>
+
+                  <.button
+                    :for={confirm <- @confirm}
+                    id={"#{@id}-confirm"}
+                    phx-click={@on_confirm}
+                    phx-disable-with
+                    class="py-2 px-3"
+                    variant={:primary}
+                  >
+                    <%= render_slot(confirm) %>
+                  </.button>
+                </div>
+              </div>
+            </.focus_wrap>
+          </div>
+        </div>
+      </div>
+    </div>
+    """
+  end
+
   ## JS Commands
 
   def show_modal(js \\ %JS{}, id) when is_binary(id) do
diff --git a/lib/oli_web/icons.ex b/lib/oli_web/icons.ex
index 8840fecf4ea..bf014d1fa2d 100644
--- a/lib/oli_web/icons.ex
+++ b/lib/oli_web/icons.ex
@@ -916,5 +916,106 @@ defmodule OliWeb.Icons do
     """
   end
 
+  attr :proficiency, :string
+
+  def proficiency(%{proficiency: "Not enough data"} = assigns) do
+    ~H"""
+    <svg
+      role="no data proficiency icon"
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        class="dark:stroke-[#747474] stroke-gray-700"
+        d="M8 8C8 7.20435 8.36875 6.44129 9.02513 5.87868C9.6815 5.31607 10.5717 5 11.5 5H12.5C13.4283 5 14.3185 5.31607 14.9749 5.87868C15.6313 6.44129 16 7.20435 16 8C16.0368 8.64925 15.8617 9.2929 15.501 9.83398C15.1402 10.3751 14.6135 10.7843 14 11C13.3865 11.2876 12.8598 11.8333 12.499 12.5547C12.1383 13.2761 11.9632 14.1343 12 15M12 19V19.01"
+        stroke="#747474"
+        stroke-width="2"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+      />
+    </svg>
+    """
+  end
+
+  def proficiency(%{proficiency: "Low"} = assigns) do
+    ~H"""
+    <svg
+      role="beginning proficiency icon"
+      width="19"
+      height="20"
+      viewBox="0 0 19 20"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect
+        class="dark:stroke-[#9B6DFF] stroke-[#7C40FF]"
+        x="1"
+        y="1"
+        width="16.9474"
+        height="18"
+        rx="2"
+        stroke-width="2"
+      />
+      <path
+        class="dark:fill-[#9B6DFF] fill-[#7C40FF]"
+        d="M0 13.334H18.3333V17.0007C18.3333 18.6575 16.9902 20.0007 15.3333 20.0007H3C1.34315 20.0007 0 18.6575 0 17.0007V13.334Z"
+      />
+    </svg>
+    """
+  end
+
+  def proficiency(%{proficiency: "Medium"} = assigns) do
+    ~H"""
+    <svg
+      role="growing proficiency icon"
+      width="19"
+      height="20"
+      viewBox="0 0 19 20"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect
+        class="dark:stroke-[#7C40FF] stroke-[#651FFF]"
+        x="1"
+        y="1"
+        width="16.9474"
+        height="18"
+        rx="2"
+        stroke-width="2"
+      />
+      <path
+        d="M0 7.5H18.3333V17C18.3333 18.6569 16.9902 20 15.3333 20H3C1.34315 20 0 18.6569 0 17V7.5Z"
+        class="dark:fill-[#7C40FF] fill-[#651FFF]"
+      />
+    </svg>
+    """
+  end
+
+  def proficiency(%{proficiency: "High"} = assigns) do
+    ~H"""
+    <svg
+      role="establishing proficiency icon"
+      width="19"
+      height="21"
+      viewBox="0 0 19 21"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect
+        class="dark:stroke-[#651FFF] dark:fill-[#651FFF] stroke-[#5104f7] fill-[#5104f7]"
+        x="1"
+        y="1.5"
+        width="16.9474"
+        height="18"
+        rx="2"
+        stroke-width="2"
+      />
+    </svg>
+    """
+  end
+
   ########## Studend Delivery Icons (end) ##########
 end
diff --git a/lib/oli_web/live/delivery/student/lesson_live.ex b/lib/oli_web/live/delivery/student/lesson_live.ex
index 26aedd9ccab..077e83b4e4d 100644
--- a/lib/oli_web/live/delivery/student/lesson_live.ex
+++ b/lib/oli_web/live/delivery/student/lesson_live.ex
@@ -9,8 +9,10 @@ defmodule OliWeb.Delivery.Student.LessonLive do
   alias Oli.Delivery.Attempts.Core.ResourceAttempt
   alias Oli.Delivery.Attempts.PageLifecycle
   alias Oli.Delivery.Attempts.PageLifecycle.FinalizationSummary
+  alias Oli.Delivery.Metrics
   alias Oli.Delivery.Page.PageContext
   alias Oli.Delivery.{Sections, Settings}
+  alias Oli.Resources
   alias Oli.Resources.Collaboration
   alias Oli.Resources.Collaboration.CollabSpaceConfig
   alias OliWeb.Common.FormatDateTime
@@ -27,7 +29,6 @@ defmodule OliWeb.Delivery.Student.LessonLive do
 
   def mount(_params, _session, %{assigns: %{view: :practice_page}} = socket) do
     %{current_user: current_user, section: section} = socket.assigns
-
     is_instructor = Sections.has_instructor_role?(current_user, section.slug)
 
     # when updating to Liveview 0.20 we should replace this with assign_async/3
@@ -49,7 +50,8 @@ defmodule OliWeb.Delivery.Student.LessonLive do
      socket
      |> assign_html_and_scripts()
      |> annotations_assigns(socket.assigns.page_context, is_instructor)
-     |> assign(is_instructor: is_instructor)}
+     |> assign(is_instructor: is_instructor)
+     |> assign_objectives()}
   end
 
   def mount(
@@ -62,7 +64,8 @@ defmodule OliWeb.Delivery.Student.LessonLive do
     {:ok,
      socket
      |> assign_html_and_scripts()
-     |> assign(begin_attempt?: false)}
+     |> assign(begin_attempt?: false)
+     |> assign_objectives()}
   end
 
   def mount(
@@ -93,7 +96,8 @@ defmodule OliWeb.Delivery.Student.LessonLive do
     {:ok,
      socket
      |> assign_scripts()
-     |> assign(begin_attempt?: false)}
+     |> assign(begin_attempt?: false)
+     |> assign_objectives()}
   end
 
   def handle_event("begin_attempt", %{"password" => password}, socket)
@@ -633,6 +637,7 @@ defmodule OliWeb.Delivery.Student.LessonLive do
           page_context={@page_context}
           ctx={@ctx}
           index={@current_page["index"]}
+          objectives={@objectives}
           container_label={Utils.get_container_label(@current_page["id"], @section)}
         />
       </:header>
@@ -695,6 +700,7 @@ defmodule OliWeb.Delivery.Student.LessonLive do
           <.page_header
             page_context={@page_context}
             ctx={@ctx}
+            objectives={@objectives}
             index={@current_page["index"]}
             container_label={Utils.get_container_label(@current_page["id"], @section)}
           />
@@ -726,6 +732,7 @@ defmodule OliWeb.Delivery.Student.LessonLive do
           <.page_header
             page_context={@page_context}
             ctx={@ctx}
+            objectives={@objectives}
             index={@current_page["index"]}
             container_label={Utils.get_container_label(@current_page["id"], @section)}
           />
@@ -794,6 +801,7 @@ defmodule OliWeb.Delivery.Student.LessonLive do
           <.page_header
             page_context={@page_context}
             ctx={@ctx}
+            objectives={@objectives}
             index={@current_page["index"]}
             container_label={Utils.get_container_label(@current_page["id"], @section)}
           />
@@ -847,14 +855,15 @@ defmodule OliWeb.Delivery.Student.LessonLive do
     <.password_attempt_modal />
 
     <div class="flex pb-20 flex-col w-full items-center gap-15 flex-1 overflow-auto">
-      <div class="flex-1 max-w-[720px] pt-20 pb-10 mx-6 flex-col justify-start items-center gap-10 inline-flex">
+      <div class="flex-1 max-w-[720px] pt-20 pb-10 mx-6 flex-col justify-start items-center inline-flex">
         <.page_header
           page_context={@page_context}
           ctx={@ctx}
+          objectives={@objectives}
           index={@current_page["index"]}
           container_label={Utils.get_container_label(@current_page["id"], @section)}
         />
-        <div class="self-stretch h-[0px] opacity-80 dark:opacity-20 bg-white border border-gray-200">
+        <div class="self-stretch h-[0px] opacity-80 dark:opacity-20 bg-white border border-gray-200 mb-10">
         </div>
         <.attempts_summary
           page_context={@page_context}
@@ -1158,6 +1167,40 @@ defmodule OliWeb.Delivery.Student.LessonLive do
     )
   end
 
+  defp assign_objectives(socket) do
+    %{page_context: %{page: page}, current_user: current_user, section: section} =
+      socket.assigns
+
+    objectives =
+      case page.objectives["attached"] do
+        objective_ids when objective_ids in [nil, []] ->
+          []
+
+        objective_resource_ids ->
+          student_proficiency_per_learning_objective =
+            Metrics.proficiency_for_student_per_learning_objective(
+              section,
+              current_user.id
+            )
+
+          Resources.get_revisions_by_resource_id(objective_resource_ids)
+          |> Enum.map(fn rev ->
+            %{
+              resource_id: rev.resource_id,
+              title: rev.title,
+              proficiency:
+                Map.get(
+                  student_proficiency_per_learning_objective,
+                  rev.resource_id,
+                  "Not enough data"
+                )
+            }
+          end)
+      end
+
+    assign(socket, objectives: objectives)
+  end
+
   defp get_max_attempts(%{effective_settings: %{max_attempts: 0}} = _page_context),
     do: "unlimited"
 
diff --git a/lib/oli_web/live/delivery/student/review_live.ex b/lib/oli_web/live/delivery/student/review_live.ex
index 10ed81cf4e5..431170b5d91 100644
--- a/lib/oli_web/live/delivery/student/review_live.ex
+++ b/lib/oli_web/live/delivery/student/review_live.ex
@@ -9,7 +9,9 @@ defmodule OliWeb.Delivery.Student.ReviewLive do
 
   alias Oli.Delivery.Attempts.PageLifecycle
   alias Oli.Delivery.Page.PageContext
+  alias Oli.Delivery.Metrics
   alias Oli.Publishing.DeliveryResolver, as: Resolver
+  alias Oli.Resources
   alias OliWeb.Delivery.Student.Utils
 
   def mount(
@@ -31,6 +33,7 @@ defmodule OliWeb.Delivery.Student.ReviewLive do
         |> assign(page_context: page_context)
         |> assign(page_revision: page_revision)
         |> assign_html_and_scripts()
+        |> assign_objectives()
       else
         socket
         |> put_flash(:error, "You are not allowed to review this attempt.")
@@ -40,6 +43,40 @@ defmodule OliWeb.Delivery.Student.ReviewLive do
     {:ok, socket}
   end
 
+  defp assign_objectives(socket) do
+    %{page_context: %{page: page}, current_user: current_user, section: section} =
+      socket.assigns
+
+    objectives =
+      case page.objectives["attached"] do
+        objective_ids when objective_ids in [nil, []] ->
+          []
+
+        objective_resource_ids ->
+          student_proficiency_per_learning_objective =
+            Metrics.proficiency_for_student_per_learning_objective(
+              section,
+              current_user.id
+            )
+
+          Resources.get_revisions_by_resource_id(objective_resource_ids)
+          |> Enum.map(fn rev ->
+            %{
+              resource_id: rev.resource_id,
+              title: rev.title,
+              proficiency:
+                Map.get(
+                  student_proficiency_per_learning_objective,
+                  rev.resource_id,
+                  "Not enough data"
+                )
+            }
+          end)
+      end
+
+    assign(socket, objectives: objectives)
+  end
+
   defp review_allowed?(page_context),
     do: page_context.effective_settings.review_submission == :allow
 
@@ -59,6 +96,7 @@ defmodule OliWeb.Delivery.Student.ReviewLive do
             page_context={@page_context}
             ctx={@ctx}
             index={@current_page["index"]}
+            objectives={@objectives}
             container_label={Utils.get_container_label(@current_page["id"], @section)}
           />
           <div id="eventIntercept" phx-update="ignore" class="content" role="page_content">
diff --git a/lib/oli_web/live/delivery/student/utils.ex b/lib/oli_web/live/delivery/student/utils.ex
index 3e51dca9677..30a737cdbeb 100644
--- a/lib/oli_web/live/delivery/student/utils.ex
+++ b/lib/oli_web/live/delivery/student/utils.ex
@@ -11,18 +11,21 @@ defmodule OliWeb.Delivery.Student.Utils do
   alias Oli.Delivery.Sections
   alias Oli.Rendering.Page
   alias OliWeb.Common.FormatDateTime
+  alias OliWeb.Components.Modal
   alias OliWeb.Icons
   alias Oli.Publishing.DeliveryResolver, as: Resolver
+  alias Phoenix.LiveView.JS
 
   attr :page_context, Oli.Delivery.Page.PageContext
   attr :ctx, OliWeb.Common.SessionContext
+  attr :objectives, :list
   attr :index, :string
   attr :container_label, :string
   attr :has_assignments?, :boolean
 
   def page_header(assigns) do
     ~H"""
-    <div id="page_header" class="flex-col justify-start items-start gap-9 flex w-full">
+    <div id="page_header" class="flex-col justify-start items-start gap-9 flex w-full mb-16">
       <div class="flex-col justify-start items-start gap-3 flex w-full">
         <div class="self-stretch flex-col justify-start items-start flex">
           <div class="self-stretch justify-between items-center inline-flex">
@@ -104,28 +107,44 @@ defmodule OliWeb.Delivery.Student.Utils do
         </div>
       </div>
       <div
-        :if={@page_context.objectives not in [nil, []]}
-        class="flex-col justify-start items-start gap-3 flex w-full"
+        :if={@objectives != []}
+        class="flex-col justify-start items-start gap-3 flex w-full mt-4"
         role="page objectives"
       >
-        <div class="self-stretch justify-start items-start gap-6 inline-flex">
-          <div class="opacity-80 dark:text-white text-sm font-bold font-['Open Sans'] uppercase tracking-wider">
-            Learning objectives
+        <div class="self-stretch justify-start items-start gap-6 inline-flex mb-6">
+          <div>
+            <span class="text-neutral-700 dark:text-neutral-300 text-base font-bold font-['Inter'] leading-normal">
+              LEARNING OBJECTIVES &
+            </span>
+            <span
+              phx-click={Modal.show_modal("proficiency_explanation_modal")}
+              class="text-blue-600 text-base font-bold font-['Inter'] leading-normal hover:underline hover:underline-offset-2 cursor-pointer"
+            >
+              PROFICIENCY
+            </span>
+            <div class="h-0">
+              <.proficiency_explanation_modal />
+            </div>
           </div>
-          <div class="hidden text-blue-500 text-sm font-semibold font-['Open Sans']">View More</div>
         </div>
         <div
-          :for={{objective, index} <- Enum.with_index(@page_context.objectives, 1)}
-          class="self-stretch flex-col justify-start items-start flex"
+          :for={{objective, index} <- Enum.with_index(@objectives, 1)}
+          class="self-stretch flex-col justify-start items-start flex ml-6"
           role={"objective #{index}"}
         >
-          <div class="self-stretch py-1 justify-start items-start inline-flex">
-            <div class="grow shrink basis-0 h-6 justify-start items-start flex">
-              <div class="w-[30px] opacity-40 dark:text-white text-xs font-bold font-['Open Sans'] leading-normal">
-                L<%= index %>
-              </div>
-              <div class="grow shrink basis-0 opacity-80 dark:text-white text-sm font-normal font-['Open Sans'] leading-normal">
-                <%= objective %>
+          <div class="relative h-[21px] justify-start items-center gap-[19px] inline-flex">
+            <.proficiency_icon_with_tooltip objective={objective} />
+            <div class="justify-start items-start gap-3.5 flex">
+              <div class="justify-start items-start gap-[17px] flex">
+                <div class="w-5 text-neutral-800 dark:text-neutral-500 text-sm font-bold font-['Inter'] leading-[21px]">
+                  L<%= index %>
+                </div>
+                <div
+                  role={"objective #{index} title"}
+                  class="text-stone-700 dark:text-stone-300 text-sm font-normal font-['Open Sans'] leading-[21px]"
+                >
+                  <%= objective.title %>
+                </div>
               </div>
             </div>
           </div>
@@ -135,6 +154,85 @@ defmodule OliWeb.Delivery.Student.Utils do
     """
   end
 
+  def proficiency_explanation_modal(assigns) do
+    assigns =
+      assign(assigns, %{
+        proficiency_levels: [
+          {"Not enough data", "Not enough information",
+           "You haven’t completed enough activities for the system to calculate learning proficiency."},
+          {"Low", "Beginning Proficiency",
+           "You’re beginning to understand key ideas, but there is room to grow."},
+          {"Medium", "Growing Proficiency",
+           "Your understanding and skills are clearly strengthening and expanding."},
+          {"High", "Establishing Proficiency",
+           "You know the material well enough to apply it in different contexts."}
+        ]
+      })
+
+    ~H"""
+    <Modal.student_delivery_modal
+      id="proficiency_explanation_modal"
+      class="lg:!w-3/4 xl:!w-2/3"
+      body_class=""
+    >
+      <:title>Measuring Learning Proficiency</:title>
+      <:subtitle>
+        This course contains several learning objectives. As you continue the course, you will receive an estimate of your understanding of each objective. This estimate takes into account the activities you complete on each page.
+      </:subtitle>
+      <div class="mb-11 text-zinc-700 dark:text-white text-base font-bold font-['Inter'] leading-normal">
+        LEARNING PROFICIENCY SCALE
+      </div>
+      <div class="flex-col justify-start items-center gap-[50px] flex">
+        <div
+          :for={{proficiency, name, description} <- @proficiency_levels}
+          class="flex-col justify-start items-start gap-[15px] flex w-full"
+        >
+          <div class="justify-start items-start gap-2.5 inline-flex">
+            <div class="mt-[2px] ml-1 w-6 h-6 scale-125">
+              <Icons.proficiency proficiency={proficiency} />
+            </div>
+            <div class="text-zinc-700 dark:text-white text-base font-bold font-['Inter'] leading-normal">
+              <%= name %>
+            </div>
+          </div>
+          <div class="text-zinc-700 dark:text-white text-base font-normal font-['Inter'] leading-normal">
+            <%= description %>
+          </div>
+        </div>
+      </div>
+    </Modal.student_delivery_modal>
+    """
+  end
+
+  attr :objective, :map
+
+  defp proficiency_icon_with_tooltip(assigns) do
+    ~H"""
+    <div
+      class="absolute top-0 left-0 z-10 w-6 h-6 cursor-pointer"
+      xphx-mouseover={JS.show(to: "#objective_#{@objective.resource_id}_tooltip")}
+      xphx-mouseout={JS.hide(to: "#objective_#{@objective.resource_id}_tooltip")}
+    >
+    </div>
+    <div class="w-6 h-6 flex items-center justify-center">
+      <Icons.proficiency proficiency={@objective.proficiency} />
+    </div>
+    <div
+      id={"objective_#{@objective.resource_id}_tooltip"}
+      class="hidden absolute h-[57px] px-6 pt-[15px] pb-6 -top-[20px] -left-6 text-gray-800 dark:text-gray-700 text-base font-normal font-['Inter'] leading-normal bg-gray-300 dark:bg-white rounded-md border-2 border-gray-700 flex-col justify-start items-start gap-4 -translate-x-full"
+    >
+      <%= proficiency_to_text(@objective.proficiency) %>
+      <div class="absolute h-[40px] w-2 bg-gray-300 dark:bg-white top-2 right-0 z-20"></div>
+      <Icons.filled_chevron_up class="absolute -right-[13px] top-[16px] fill-gray-300 dark:fill-white z-10 rotate-90 scale-150 stroke-1.5 stroke-gray-700" />
+    </div>
+    """
+  end
+
+  defp proficiency_to_text("Not enough data"), do: "Not enough information"
+  defp proficiency_to_text("Low"), do: "Beginning Proficiency"
+  defp proficiency_to_text("Medium"), do: "Growing Proficiency"
+  defp proficiency_to_text("High"), do: "Establishing Proficiency"
+
   attr :scripts, :list
   attr :user_token, :string
 
diff --git a/test/oli/analytics/summary/metrics_v2_test.exs b/test/oli/analytics/summary/metrics_v2_test.exs
index 8f17f57167a..e84ebcbdee8 100644
--- a/test/oli/analytics/summary/metrics_v2_test.exs
+++ b/test/oli/analytics/summary/metrics_v2_test.exs
@@ -1,7 +1,6 @@
 defmodule Oli.Analytics.Summary.MetricsV2Test do
   use Oli.DataCase
 
-  alias Oli.Analytics.Summary
   alias Oli.Delivery.Sections.ContainedPage
   alias Oli.Delivery.Metrics
 
@@ -12,6 +11,7 @@ defmodule Oli.Analytics.Summary.MetricsV2Test do
         |> Seeder.create_section()
         |> Seeder.add_objective("objective one", :o1)
         |> Seeder.add_objective("objective two", :o2)
+        |> Seeder.add_objective("objective three", :o3)
         |> Seeder.add_activity(%{title: "one", content: %{}}, :a1)
         |> Seeder.add_activity(%{title: "two", content: %{}}, :a2)
         |> Seeder.add_user(%{}, :user1)
@@ -131,22 +131,36 @@ defmodule Oli.Analytics.Summary.MetricsV2Test do
       assert Map.keys(results) |> Enum.count() == 2
       assert %{^user1_id => "High", ^user2_id => "Low"} = results
     end
-  end
 
-  defp add_resource_summary([prj, pub, section, user, resource, part, type, nc, na, nh, nfa, nfac]) do
-    Summary.create_resource_summary(%{
-      project_id: prj,
-      publication_id: pub,
-      section_id: section,
-      user_id: user,
-      resource_id: resource,
-      part_id: part,
-      resource_type_id: type,
-      num_correct: nc,
-      num_attempts: na,
-      num_hints: nh,
-      num_first_attempts: nfa,
-      num_first_attempts_correct: nfac
-    })
+    test "proficiency_for_student_per_learning_objective/2", %{
+      user1: user1,
+      user2: user2,
+      section: section,
+      o1: o1,
+      o2: o2,
+      o3: o3
+    } do
+      objective_type_id = Oli.Resources.ResourceType.id_for_objective()
+      {:ok, section} = Oli.Delivery.Sections.update_section(section, %{analytics_version: :v2})
+
+      id = o1.resource.id
+      id2 = o2.resource.id
+      id3 = o3.resource.id
+
+      [
+        [-1, -1, section.id, user1.id, id, nil, objective_type_id, 2, 6, 1, 1, 0],
+        [-1, -1, section.id, user1.id, id2, nil, objective_type_id, 2, 6, 1, 3, 2],
+        [-1, -1, section.id, user1.id, id3, nil, objective_type_id, 2, 6, 1, 3, 3],
+        [-1, -1, section.id, user2.id, id, nil, objective_type_id, 2, 4, 0, 1, 1],
+        [-1, -1, section.id, user2.id, id2, nil, objective_type_id, 2, 4, 0, 4, 2]
+      ]
+      |> Enum.each(fn v -> add_resource_summary(v) end)
+
+      results = Metrics.proficiency_for_student_per_learning_objective(section, user1.id)
+      assert Map.keys(results) |> Enum.count() == 3
+      assert Map.get(results, id) == "Low"
+      assert Map.get(results, id2) == "Medium"
+      assert Map.get(results, id3) == "High"
+    end
   end
 end
diff --git a/test/oli/delivery/metrics/learning_proficiency_test.exs b/test/oli/delivery/metrics/learning_proficiency_test.exs
index 26a19e06d24..2e4c6c19ef0 100644
--- a/test/oli/delivery/metrics/learning_proficiency_test.exs
+++ b/test/oli/delivery/metrics/learning_proficiency_test.exs
@@ -661,5 +661,27 @@ defmodule Oli.Delivery.Metrics.LearningProficiencyTest do
       assert proficiency_for_student_per_page[page_2.resource_id] == "Low"
       assert proficiency_for_student_per_page[page_3.resource_id] == "Low"
     end
+
+    test "proficiency_for_student_per_learning_objective/2 calculates correctly", %{
+      section: section,
+      student_1: student_1,
+      page_1: page_1,
+      page_2: page_2,
+      page_3: page_3,
+      page_1_objective: page_1_obj,
+      page_2_objective: page_2_obj,
+      page_3_objective: page_3_obj
+    } do
+      set_snapshot(section, page_1.resource, page_1_obj.resource, student_1, true)
+      set_snapshot(section, page_2.resource, page_2_obj.resource, student_1, false)
+      set_snapshot(section, page_3.resource, page_3_obj.resource, student_1, false)
+
+      proficiency_for_student_per_learning_objective =
+        Metrics.proficiency_for_student_per_learning_objective(section, student_1.id)
+
+      assert proficiency_for_student_per_learning_objective[page_1_obj.resource_id] == "High"
+      assert proficiency_for_student_per_learning_objective[page_2_obj.resource_id] == "Low"
+      assert proficiency_for_student_per_learning_objective[page_3_obj.resource_id] == "Low"
+    end
   end
 end
diff --git a/test/oli_web/live/delivery/student/lesson_live_test.exs b/test/oli_web/live/delivery/student/lesson_live_test.exs
index 0cfb47f997c..dbfdde54f17 100644
--- a/test/oli_web/live/delivery/student/lesson_live_test.exs
+++ b/test/oli_web/live/delivery/student/lesson_live_test.exs
@@ -81,6 +81,18 @@ defmodule OliWeb.Delivery.Student.LessonLiveTest do
         title: "this is the second objective"
       )
 
+    objective_3_revision =
+      insert(:revision,
+        resource_type_id: ResourceType.get_id_by_type("objective"),
+        title: "this is the third objective"
+      )
+
+    objective_4_revision =
+      insert(:revision,
+        resource_type_id: ResourceType.get_id_by_type("objective"),
+        title: "this is the forth objective"
+      )
+
     ## pages...
     page_1_revision =
       insert(:revision,
@@ -140,7 +152,12 @@ defmodule OliWeb.Delivery.Student.LessonLiveTest do
         title: "Page 2",
         duration_minutes: 15,
         objectives: %{
-          "attached" => [objective_1_revision.resource_id, objective_2_revision.resource_id]
+          "attached" => [
+            objective_1_revision.resource_id,
+            objective_2_revision.resource_id,
+            objective_3_revision.resource_id,
+            objective_4_revision.resource_id
+          ]
         }
       )
 
@@ -241,6 +258,8 @@ defmodule OliWeb.Delivery.Student.LessonLiveTest do
       [
         objective_1_revision,
         objective_2_revision,
+        objective_3_revision,
+        objective_4_revision,
         page_1_revision,
         exploration_1_revision,
         graded_adaptive_page_revision,
@@ -318,6 +337,10 @@ defmodule OliWeb.Delivery.Student.LessonLiveTest do
 
     %{
       section: section,
+      objective_1: objective_1_revision,
+      objective_2: objective_2_revision,
+      objective_3: objective_3_revision,
+      objective_4: objective_4_revision,
       page_1: page_1_revision,
       exploration_1: exploration_1_revision,
       graded_adaptive_page_revision: graded_adaptive_page_revision,
@@ -795,8 +818,100 @@ defmodule OliWeb.Delivery.Student.LessonLiveTest do
       assert has_element?(view, ~s{div[role="page title"]}, "Page 2")
       assert has_element?(view, ~s{div[role="page read time"]}, "15")
       assert has_element?(view, ~s{div[role="page schedule"]}, "Tue Nov 14, 2023")
-      assert has_element?(view, ~s{div[role="objective 1"]}, "this is the first objective")
-      assert has_element?(view, ~s{div[role="objective 2"]}, "this is the second objective")
+    end
+
+    test "can see proficiency explanation modal", %{
+      conn: conn,
+      user: user,
+      section: section,
+      page_2: page_2
+    } do
+      Sections.enroll(user.id, section.id, [ContextRoles.get_role(:context_learner)])
+      Sections.mark_section_visited_for_student(section, user)
+
+      {:ok, view, _html} = live(conn, Utils.lesson_live_path(section.slug, page_2.slug))
+
+      # if we render_click() on PROFICIENCY to show the modal, a JS command should be triggered.
+      # But, since the phx-click does not have any push command, we get "no push command found within JS commands".
+      # The workaround is to directly assert on the content of the modal
+
+      assert has_element?(
+               view,
+               "#proficiency_explanation_modal h1",
+               "Measuring Learning Proficiency"
+             )
+
+      assert has_element?(
+               view,
+               "#proficiency_explanation_modal p",
+               "This course contains several learning objectives. As you continue the course, you will receive an estimate of your understanding of each objective. This estimate takes into account the activities you complete on each page."
+             )
+    end
+
+    test "can see learning objectives and proficiency on page header", %{
+      conn: conn,
+      user: user,
+      section: section,
+      page_2: page_2,
+      objective_1: objective_1,
+      objective_2: objective_2,
+      objective_3: objective_3,
+      objective_4: objective_4
+    } do
+      Sections.enroll(user.id, section.id, [ContextRoles.get_role(:context_learner)])
+      Sections.mark_section_visited_for_student(section, user)
+      o1 = objective_1.resource_id
+      o2 = objective_2.resource_id
+      o3 = objective_3.resource_id
+      o4 = objective_4.resource_id
+
+      objective_type_id = Oli.Resources.ResourceType.id_for_objective()
+
+      [
+        # objective records
+        [-1, -1, section.id, user.id, o1, nil, objective_type_id, 2, 6, 1, 4, 1],
+        [-1, -1, section.id, user.id, o2, nil, objective_type_id, 2, 6, 1, 4, 3],
+        [-1, -1, section.id, user.id, o3, nil, objective_type_id, 2, 6, 1, 4, 4]
+      ]
+      |> Enum.each(&add_resource_summary(&1))
+
+      {:ok, view, _html} = live(conn, Utils.lesson_live_path(section.slug, page_2.slug))
+
+      assert has_element?(view, ~s{div[role="objective 1 title"]}, "this is the first objective")
+
+      assert has_element?(
+               view,
+               ~s{div[role="objective 1"] svg[role="beginning proficiency icon"]}
+             )
+
+      assert has_element?(view, ~s{div[id="objective_#{o1}_tooltip"]}, "Beginning Proficiency")
+
+      assert has_element?(view, ~s{div[role="objective 2 title"]}, "this is the second objective")
+
+      assert has_element?(
+               view,
+               ~s{div[role="objective 2"] svg[role="growing proficiency icon"]}
+             )
+
+      assert has_element?(view, ~s{div[id="objective_#{o2}_tooltip"]}, "Growing Proficiency")
+
+      assert has_element?(view, ~s{div[role="objective 3 title"]}, "this is the third objective")
+
+      assert has_element?(
+               view,
+               ~s{div[role="objective 3"] svg[role="establishing proficiency icon"]}
+             )
+
+      assert has_element?(view, ~s{div[id="objective_#{o3}_tooltip"]}, "Establishing Proficiency")
+
+      assert has_element?(view, ~s{div[role="objective 4 title"]}, "this is the forth objective")
+
+      assert has_element?(
+               view,
+               ~s{div[role="objective 4"] svg[role="no data proficiency icon"]}
+             )
+
+      assert has_element?(view, ~s{div[id="objective_#{o4}_tooltip"]}, "Not enough information")
     end
 
     test "can navigate between pages and updates references in the request path", %{
diff --git a/test/support/test_helpers.ex b/test/support/test_helpers.ex
index d7d011d766d..ecd566938b2 100644
--- a/test/support/test_helpers.ex
+++ b/test/support/test_helpers.ex
@@ -7,6 +7,7 @@ defmodule Oli.TestHelpers do
   alias Oli.Accounts
   alias Oli.Accounts.{Author, AuthorPreferences, User}
   alias Oli.Activities
+  alias Oli.Analytics.Summary
   alias Oli.Authoring.Course
   alias Oli.Authoring.Course.Project
   alias Oli.Delivery.Sections
@@ -3473,6 +3474,23 @@ defmodule Oli.TestHelpers do
     |> set_container_children(container_resource, container_revision, publication)
   end
 
+  def add_resource_summary([prj, pub, section, user, resource, part, type, nc, na, nh, nfa, nfac]) do
+    Summary.create_resource_summary(%{
+      project_id: prj,
+      publication_id: pub,
+      section_id: section,
+      user_id: user,
+      resource_id: resource,
+      part_id: part,
+      resource_type_id: type,
+      num_correct: nc,
+      num_attempts: na,
+      num_hints: nh,
+      num_first_attempts: nfa,
+      num_first_attempts_correct: nfac
+    })
+  end
+
   defp set_container_children(children, container, container_revision, publication) do
     {:ok, updated_revision} =
       Oli.Resources.create_revision_from_previous(container_revision, %{children: children})

From 32498051478aaab4ccd54d27525b2a10899cc336 Mon Sep 17 00:00:00 2001
From: Tomas Fernandez <80487824+tomasferok@users.noreply.github.com>
Date: Tue, 21 May 2024 15:48:05 -0300
Subject: [PATCH 2/2] [BUG FIX] [MER-2932]
 filter-insights-by-product-course-section (#4777)

* [MER-2932] correct filter in tab by page

* [MER-2932] IO inspect removed

* [MER-2932] correct remix section removing the child activities associated to a revision

* [MER-2932] remove repo all and some improvements

* merge with master

* [MER-2932] small fix

* merge with master

* [MER-2932] add test
---
 lib/oli/analytics/by_activity.ex              |  5 +-
 lib/oli/analytics/by_objective.ex             |  2 +-
 lib/oli/analytics/by_page.ex                  |  9 +-
 lib/oli/delivery/sections.ex                  | 85 +++++++++++++++++--
 lib/oli/publishing/delivery_resolver.ex       |  5 +-
 lib/oli/utils/db_seeder.ex                    | 37 +++++++-
 .../oli/delivery/analytics/analytics_test.exs |  6 +-
 .../insights/content_tab_test.exs             | 25 +++++-
 test/oli_web/live/remix_section_test.exs      | 63 +++++++++++++-
 9 files changed, 210 insertions(+), 27 deletions(-)

diff --git a/lib/oli/analytics/by_activity.ex b/lib/oli/analytics/by_activity.ex
index 28d0557c82c..217f09ad328 100644
--- a/lib/oli/analytics/by_activity.ex
+++ b/lib/oli/analytics/by_activity.ex
@@ -18,7 +18,7 @@ defmodule Oli.Analytics.ByActivity do
   defp get_base_query(project_slug, filtered_sections) do
     subquery =
       if filtered_sections != [] do
-        DeliveryResolver.revisions_filter_by_section_ids(
+        DeliveryResolver.revisions_by_section_ids(
           filtered_sections,
           ResourceType.id_for_activity()
         )
@@ -39,7 +39,6 @@ defmodule Oli.Analytics.ByActivity do
         number_of_attempts: analytics.number_of_attempts,
         relative_difficulty: analytics.relative_difficulty
       },
-      preload: [:resource_type],
-      distinct: [activity]
+      preload: [:resource_type]
   end
 end
diff --git a/lib/oli/analytics/by_objective.ex b/lib/oli/analytics/by_objective.ex
index 256ac807445..1492b375fb7 100644
--- a/lib/oli/analytics/by_objective.ex
+++ b/lib/oli/analytics/by_objective.ex
@@ -20,7 +20,7 @@ defmodule Oli.Analytics.ByObjective do
   defp get_base_query(project_slug, activity_objectives, filtered_sections) do
     subquery =
       if filtered_sections != [] do
-        DeliveryResolver.revisions_filter_by_section_ids(
+        DeliveryResolver.revisions_by_section_ids(
           filtered_sections,
           ResourceType.id_for_objective()
         )
diff --git a/lib/oli/analytics/by_page.ex b/lib/oli/analytics/by_page.ex
index 18ffcace7cf..e8a424bcc99 100644
--- a/lib/oli/analytics/by_page.ex
+++ b/lib/oli/analytics/by_page.ex
@@ -22,7 +22,7 @@ defmodule Oli.Analytics.ByPage do
   defp get_base_query(project_slug, activity_pages, filtered_sections) do
     subquery =
       if filtered_sections != [] do
-        DeliveryResolver.revisions_filter_by_section_ids(
+        DeliveryResolver.revisions_by_section_ids(
           filtered_sections,
           ResourceType.id_for_page()
         )
@@ -35,9 +35,9 @@ defmodule Oli.Analytics.ByPage do
 
     subquery_activity =
       if filtered_sections != [] do
-        DeliveryResolver.revisions_filter_by_section_ids(
+        DeliveryResolver.revisions_by_section_ids(
           filtered_sections,
-          ResourceType.id_for_page()
+          ResourceType.id_for_activity()
         )
       else
         Publishing.query_unpublished_revisions_by_type(
@@ -62,8 +62,7 @@ defmodule Oli.Analytics.ByPage do
         number_of_attempts: analytics.number_of_attempts,
         relative_difficulty: analytics.relative_difficulty
       },
-      preload: [:resource_type],
-      distinct: [activity]
+      preload: [:resource_type]
     )
   end
 
diff --git a/lib/oli/delivery/sections.ex b/lib/oli/delivery/sections.ex
index fee608f0f52..76ab3c52b59 100644
--- a/lib/oli/delivery/sections.ex
+++ b/lib/oli/delivery/sections.ex
@@ -2335,7 +2335,8 @@ defmodule Oli.Delivery.Sections do
       # create any remaining section resources which are not in the hierarchy
       create_nonstructural_section_resources(section.id, [publication_id],
         skip_resource_ids: processed_ids,
-        required_survey_resource_id: survey_id
+        required_survey_resource_id: survey_id,
+        reference_activity_ids: []
       )
 
       root_section_resource_id = section_resources_by_resource_id[root_resource_id].id
@@ -2823,10 +2824,15 @@ defmodule Oli.Delivery.Sections do
         |> select([p], p.required_survey_resource_id)
         |> Repo.one()
 
-      create_nonstructural_section_resources(section_id, publication_ids,
-        skip_resource_ids: processed_resource_ids,
-        required_survey_resource_id: survey_id
-      )
+      reference_activity_ids = build_reference_activity_ids(hierarchy.children)
+
+      if length(processed_resource_ids) != 1 do
+        create_nonstructural_section_resources(section_id, publication_ids,
+          skip_resource_ids: processed_resource_ids,
+          required_survey_resource_id: survey_id,
+          reference_activity_ids: reference_activity_ids
+        )
+      end
 
       # Rebuild section previous next index
       PreviousNextIndex.rebuild(section, hierarchy)
@@ -2838,6 +2844,62 @@ defmodule Oli.Delivery.Sections do
     end)
   end
 
+  defp build_reference_activity_ids([]), do: []
+
+  defp build_reference_activity_ids(children_activities) do
+    tuple_of_children_activities =
+      process_activity_ids(children_activities)
+
+    tuple_of_children_activities
+    |> Tuple.to_list()
+    |> Enum.concat()
+  end
+
+  defp process_activity_ids(children_activities) do
+    children_activities
+    |> Enum.reduce({[], []}, fn r, {children_unit_activities_ids, children_activities_ids} ->
+      new_children_unit_activities_ids = build_child_activity_ids_for_units(r.children)
+
+      if Map.has_key?(r.revision, :content) do
+        new_children_activities_ids = build_child_activity_ids(r.revision.content)
+
+        {children_unit_activities_ids ++ new_children_unit_activities_ids,
+         children_activities_ids ++ new_children_activities_ids}
+      else
+        {children_unit_activities_ids ++ new_children_unit_activities_ids,
+         children_activities_ids ++ []}
+      end
+    end)
+  end
+
+  defp build_child_activity_ids_for_units([]), do: []
+
+  defp build_child_activity_ids_for_units(children) do
+    children
+    |> Enum.map(fn c ->
+      if Map.has_key?(c.revision, :content) do
+        build_child_activity_ids(c.revision.content)
+      else
+        []
+      end
+    end)
+    |> List.flatten()
+  end
+
+  defp build_child_activity_ids(%{"model" => nil}), do: []
+
+  defp build_child_activity_ids(%{"model" => model}) do
+    model
+    |> Enum.reduce([], fn item, activity_ids ->
+      case item["activity_id"] do
+        nil -> activity_ids
+        activity_id -> [activity_id | activity_ids]
+      end
+    end)
+  end
+
+  defp build_child_activity_ids(_), do: []
+
   def get_contained_pages(%Section{id: section_id}) do
     from(cp in ContainedPage,
       where: cp.section_id == ^section_id
@@ -3630,10 +3692,11 @@ defmodule Oli.Delivery.Sections do
   # any that belong to the resource ids in skip_resource_ids
   defp create_nonstructural_section_resources(section_id, publication_ids,
          skip_resource_ids: skip_resource_ids,
-         required_survey_resource_id: required_survey_resource_id
+         required_survey_resource_id: required_survey_resource_id,
+         reference_activity_ids: reference_activity_ids
        ) do
     published_resources_by_resource_id =
-      MinimalHierarchy.published_resources_map(publication_ids)
+      build_published_resources_by_resource_id(publication_ids, reference_activity_ids)
 
     now = DateTime.utc_now() |> DateTime.truncate(:second)
 
@@ -3677,6 +3740,14 @@ defmodule Oli.Delivery.Sections do
     Database.batch_insert_all(SectionResource, section_resource_rows)
   end
 
+  defp build_published_resources_by_resource_id(publication_ids, []),
+    do: MinimalHierarchy.published_resources_map(publication_ids)
+
+  defp build_published_resources_by_resource_id(publication_ids, list_children_section_ids),
+    do:
+      MinimalHierarchy.published_resources_map(publication_ids)
+      |> Map.take(list_children_section_ids)
+
   def is_structural?(%Revision{resource_type_id: resource_type_id}) do
     container = ResourceType.id_for_container()
 
diff --git a/lib/oli/publishing/delivery_resolver.ex b/lib/oli/publishing/delivery_resolver.ex
index c2b575626c9..aa2b97189a7 100644
--- a/lib/oli/publishing/delivery_resolver.ex
+++ b/lib/oli/publishing/delivery_resolver.ex
@@ -263,7 +263,7 @@ defmodule Oli.Publishing.DeliveryResolver do
     |> emit([:oli, :resolvers, :delivery], :duration)
   end
 
-  def revisions_filter_by_section_ids(section_ids, resource_type_id) do
+  def revisions_by_section_ids(section_ids, resource_type_id) do
     from(sr in SectionResource,
       join: s in Section,
       on: s.id == sr.section_id,
@@ -277,7 +277,8 @@ defmodule Oli.Publishing.DeliveryResolver do
       join: rev in Revision,
       on: rev.id == pr.revision_id,
       where: rev.resource_type_id == ^resource_type_id and rev.deleted == false,
-      select: rev
+      select: rev,
+      distinct: [rev]
     )
   end
 
diff --git a/lib/oli/utils/db_seeder.ex b/lib/oli/utils/db_seeder.ex
index 05acb982a8a..b4803f74cca 100644
--- a/lib/oli/utils/db_seeder.ex
+++ b/lib/oli/utils/db_seeder.ex
@@ -198,14 +198,34 @@ defmodule Oli.Seeder do
 
     create_published_resource(publication, container_resource, container_revision)
 
+    %{resource: _activity_resource, revision: activity_revision} =
+      create_activity(
+        %{
+          activity_type_id: Activities.get_registration_by_slug("oli_short_answer").id,
+          content: %{}
+        },
+        publication,
+        project,
+        author
+      )
+
     %{resource: page1, revision: revision1} =
       create_page("Page one", publication, project, author)
 
     %{resource: page2, revision: revision2} =
       create_page("Page two", publication, project, author, create_sample_content())
 
+    %{resource: page3, revision: revision3} =
+      create_page(
+        "Page three",
+        publication,
+        project,
+        author,
+        create_activity_content(activity_revision.resource_id)
+      )
+
     container_revision =
-      attach_pages_to([page1, page2], container_resource, container_revision, publication)
+      attach_pages_to([page1, page2, page3], container_resource, container_revision, publication)
 
     {:ok, pub1} = Publishing.publish_project(project, "some changes", author.id)
 
@@ -229,8 +249,10 @@ defmodule Oli.Seeder do
     |> Map.put(:container, %{resource: container_resource, revision: container_revision})
     |> Map.put(:page1, page1)
     |> Map.put(:page2, page2)
+    |> Map.put(:page3, page3)
     |> Map.put(:revision1, revision1)
     |> Map.put(:revision2, revision2)
+    |> Map.put(:revision3, revision3)
     |> Map.put(:section, section)
   end
 
@@ -1290,6 +1312,19 @@ defmodule Oli.Seeder do
     }
   end
 
+  def create_activity_content(activity_resource_id) do
+    %{
+      "model" => [
+        %{
+          "type" => "activity-reference",
+          "activity_id" => activity_resource_id,
+          "custom" => %{}
+        }
+      ],
+      "advancedDelivery" => false
+    }
+  end
+
   def create_sample_content() do
     %{
       "model" => [
diff --git a/test/oli/delivery/analytics/analytics_test.exs b/test/oli/delivery/analytics/analytics_test.exs
index 431d04ea073..7db69586a38 100644
--- a/test/oli/delivery/analytics/analytics_test.exs
+++ b/test/oli/delivery/analytics/analytics_test.exs
@@ -100,7 +100,7 @@ defmodule Oli.Delivery.Analytics.AnalyticsTest do
     } do
       assert length(activity_query) == 4
       assert length(objective_query) == 2
-      assert length(page_query) == 4
+      assert length(page_query) == 5
     end
   end
 
@@ -310,7 +310,7 @@ defmodule Oli.Delivery.Analytics.AnalyticsTest do
       # Parent course should still have analytics after duplicating project
       assert Enum.count(objective_query) == 2
       assert Enum.count(activity_query) == 4
-      assert Enum.count(page_query) == 4
+      assert Enum.count(page_query) == 5
 
       # Duplicated course should not have analytics
 
@@ -341,7 +341,7 @@ defmodule Oli.Delivery.Analytics.AnalyticsTest do
 
       # 3 pages with no analytics
       page_insights = ByPage.query_against_project_slug(duplicated.slug, [])
-      assert Enum.count(page_insights) == 1
+      assert Enum.count(page_insights) == 3
 
       assert Enum.all?(page_insights, fn obj ->
                Enum.all?(insights_from_struct.(obj), &is_nil(&1))
diff --git a/test/oli_web/live/delivery/instructor_dashboard/insights/content_tab_test.exs b/test/oli_web/live/delivery/instructor_dashboard/insights/content_tab_test.exs
index 0e98976d2af..ec0bc1dee42 100644
--- a/test/oli_web/live/delivery/instructor_dashboard/insights/content_tab_test.exs
+++ b/test/oli_web/live/delivery/instructor_dashboard/insights/content_tab_test.exs
@@ -336,7 +336,15 @@ defmodule OliWeb.Delivery.InstructorDashboard.ContentTabTest do
            instructor: instructor,
            conn: conn
          } do
-      %{page1: page1, revision1: revision1, page2: page2, revision2: revision2, section: section} =
+      %{
+        page1: page1,
+        revision1: revision1,
+        page2: page2,
+        revision2: revision2,
+        page3: page3,
+        revision3: revision3,
+        section: section
+      } =
         Oli.Seeder.base_project_with_pages()
 
       user_1 = insert(:user, %{given_name: "Lionel", family_name: "Messi"})
@@ -362,6 +370,11 @@ defmodule OliWeb.Delivery.InstructorDashboard.ContentTabTest do
       set_progress(section.id, page2.id, user_3.id, 0.8, revision2)
       set_progress(section.id, page2.id, user_4.id, 0.8, revision2)
 
+      set_progress(section.id, page3.id, user_1.id, 0.9, revision3)
+      set_progress(section.id, page3.id, user_2.id, 0.6, revision3)
+      set_progress(section.id, page3.id, user_3.id, 0, revision3)
+      set_progress(section.id, page3.id, user_4.id, 0.3, revision3)
+
       {:ok, view, _html} = live(conn, live_view_content_route(section.slug))
 
       progress =
@@ -371,7 +384,7 @@ defmodule OliWeb.Delivery.InstructorDashboard.ContentTabTest do
         |> Floki.find(~s{.instructor_dashboard_table tr [data-progress-check]})
         |> Enum.map(fn div_tag -> Floki.text(div_tag) |> String.trim() end)
 
-      assert progress == ["45%", "80%"]
+      assert progress == ["45%", "80%", "45%"]
 
       ### links to students tab with page_id as url param
       links =
@@ -397,6 +410,14 @@ defmodule OliWeb.Delivery.InstructorDashboard.ContentTabTest do
                  :insights,
                  :content,
                  %{page_id: page2.id}
+               ),
+               Routes.live_path(
+                 OliWeb.Endpoint,
+                 OliWeb.Delivery.InstructorDashboard.InstructorDashboardLive,
+                 section.slug,
+                 :insights,
+                 :content,
+                 %{page_id: page3.id}
                )
              ]
 
diff --git a/test/oli_web/live/remix_section_test.exs b/test/oli_web/live/remix_section_test.exs
index b0ce5487d1a..2f6349c8e84 100644
--- a/test/oli_web/live/remix_section_test.exs
+++ b/test/oli_web/live/remix_section_test.exs
@@ -14,6 +14,41 @@ defmodule OliWeb.RemixSectionLiveTest do
   describe "remix section as admin" do
     setup [:setup_admin_session]
 
+    test "remix section remove a pege and verify activities of this page has been deleted", %{
+      conn: conn
+    } do
+      map =
+        Seeder.base_project_with_pages()
+
+      assert Sections.get_section_resource(map.section.id, map.revision3.resource_id).resource_id ==
+               map.revision3.resource_id
+
+      conn =
+        get(
+          conn,
+          Routes.live_path(OliWeb.Endpoint, OliWeb.Delivery.RemixSection, map.section.slug)
+        )
+
+      {:ok, view, _html} = live(conn)
+
+      node_children_uuids =
+        view
+        |> render()
+        |> Floki.parse_fragment!()
+        |> Floki.find(~s{button[phx-click="show_remove_modal"]})
+        |> Floki.attribute("phx-value-uuid")
+
+      open_modal_and_confirm_removal(node_children_uuids, view)
+
+      assert render(view) =~ "<p>There&#39;s nothing here.</p>"
+
+      view
+      |> element("#save")
+      |> render_click()
+
+      assert Sections.get_section_resource(map.section.id, map.revision3.resource_id) == nil
+    end
+
     test "mount as admin", %{
       conn: conn,
       map: %{
@@ -773,7 +808,26 @@ defmodule OliWeb.RemixSectionLiveTest do
   end
 
   defp setup_admin_session(%{conn: conn}) do
-    map = Seeder.base_project_with_resource4()
+    map =
+      Seeder.base_project_with_resource4()
+      |> Seeder.add_activity(%{title: "one"}, :publication, :project, :author, :a1)
+
+    attrs = %{
+      title: "page1",
+      content: %{
+        "model" => [
+          %{
+            "type" => "activity-reference",
+            "activity_id" => Map.get(map, :a1).resource.id,
+            "custom" => %{}
+          }
+        ],
+        "advancedDelivery" => false
+      },
+      graded: false
+    }
+
+    map = Seeder.add_page(map, attrs, :p1)
 
     admin = author_fixture(%{system_role_id: Oli.Accounts.SystemRole.role_id().system_admin})
 
@@ -801,7 +855,9 @@ defmodule OliWeb.RemixSectionLiveTest do
 
     author = insert(:author, %{email: "my_custom@email.com"})
 
-    proj_1 = insert(:project, title: "Project 1", authors: [author])
+    proj_1 =
+      insert(:project, title: "Project 1", authors: [author])
+
     proj_2 = insert(:project, title: "Project 2", authors: [author])
     proj_3 = insert(:project, title: "Project 3", authors: [author])
     proj_4 = insert(:project, title: "Project 4", authors: [author])
@@ -918,7 +974,8 @@ defmodule OliWeb.RemixSectionLiveTest do
   end
 
   defp setup_instructor_session(%{conn: conn}) do
-    map = Seeder.base_project_with_resource4()
+    map =
+      Seeder.base_project_with_resource4()
 
     {:ok, instructor} =
       Accounts.update_user_platform_roles(