Skip to content

Commit 5eec845

Browse files
saraburns1saraburns1
committed
feat!: final round of cleanup
1 parent 37ea31a commit 5eec845

File tree

71 files changed

+20933
-18739
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+20933
-18739
lines changed

‎macros/a_tag.sql‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎macros/format_problem_number_location.sql‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
{% macro format_problem_number_location(
2-
object_id_var, display_name_with_location_var
3-
) %}
1+
{% macro format_problem_location(object_id_var, display_name_with_location_var) %}
42
substring(
53
regexpExtract({{ object_id_var }}, '(@problem\+block@[\w\d][^_\/]*)(_\d)?', 2),
64
2
@@ -14,6 +12,23 @@
1412
),
1513
':'
1614
) as _problem_number,
17-
concat(_problem_number, '_', _problem_id_or_1) as problem_number,
18-
concat(problem_number, ' - ', _problem_with_name[2]) as problem_name_location
15+
concat(_problem_number, '_', _problem_id_or_1) as problem_location,
16+
concat(problem_location, ' - ', _problem_with_name[2]) as problem_name_location
17+
{% endmacro %}
18+
19+
{% macro format_object_location(display_name_with_location_var) %}
20+
arrayStringConcat(
21+
arrayMap(
22+
x -> (leftPad(x, 2, char(917768))),
23+
splitByString(
24+
':', splitByString(' - ', {{ display_name_with_location_var }})[1]
25+
)
26+
),
27+
':'
28+
) as object_location,
29+
concat(
30+
object_location,
31+
' - ',
32+
splitByString(' - ', {{ display_name_with_location_var }})[2]
33+
) as object_name_location
1934
{% endmacro %}

‎macros/get_bucket.sql‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎macros/get_problem_id.sql‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎macros/items_per_subsection.sql‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎macros/parse_section_codes.sql‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎macros/remove_deprecated_models.sql‎

Lines changed: 47 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -30,64 +30,67 @@
3030

3131
-- https://github.com/openedx/aspects-dbt/pull/111/
3232
{% set models_to_drop = [
33-
("view", reporting, "fact_completions"),
34-
("view", xapi, "fact_forum_interactions"),
35-
("view", reporting, "fact_grades"),
36-
("view", reporting, "learner_summary"),
37-
("view", reporting, "fact_navigation_dropoff"),
38-
("view", reporting, "fact_learner_problem_summary"),
39-
("view", reporting, "fact_problem_engagement"),
40-
("view", reporting, "fact_problem_engagement_per_subsection"),
41-
("view", reporting, "fact_problem_responses_extended"),
42-
("view", reporting, "dim_at_risk_learners"),
43-
("view", reporting, "fact_transcript_usage"),
44-
("view", reporting, "fact_watched_video_segments"),
45-
("view", reporting, "dim_course_blocks_extended"),
46-
("view", reporting, "fact_navigation"),
47-
("view", reporting, "int_pages_per_subsection"),
48-
("view", reporting, "int_problems_per_subsection"),
49-
("view", reporting, "fact_problem_responses"),
50-
("view", reporting, "int_problem_hints"),
51-
("view", reporting, "int_problem_results"),
52-
("view", reporting, "most_recent_course_tags"),
53-
("view", reporting, "fact_student_status"),
54-
("view", reporting, "watched_video_duration"),
55-
("view", reporting, "fact_video_plays"),
56-
("view", reporting, "fact_watched_video_duration"),
57-
("view", reporting, "fact_learner_response_attempts"),
33+
("dictionary", event_sink, "course_block_names"),
34+
("dictionary", event_sink, "course_names"),
35+
("dictionary", event_sink, "course_tags"),
5836
("mv", event_sink, "course_enrollment"),
5937
("mv", event_sink, "course_relationships"),
60-
("mv", xapi, "forum_events"),
61-
("mv", reporting, "video_transcript_events"),
62-
("mv", reporting, "fact_learner_course_grade"),
63-
("mv", reporting, "fact_learner_course_status"),
6438
("mv", event_sink, "most_recent_course_blocks"),
65-
("mv", reporting, "fact_enrollment_status"),
6639
("mv", event_sink, "most_recent_object_tags"),
6740
("mv", event_sink, "most_recent_tags"),
6841
("mv", event_sink, "most_recent_taxonomies"),
42+
("mv", reporting, "dim_learner_first_response"),
43+
("mv", reporting, "dim_learner_first_success_response"),
44+
("mv", reporting, "dim_learner_last_course_visit"),
45+
("mv", reporting, "dim_learner_most_recent_course_grade"),
46+
("mv", reporting, "dim_learner_response_attempt"),
47+
("mv", reporting, "fact_enrollment_status"),
48+
("mv", reporting, "fact_learner_course_grade"),
49+
("mv", reporting, "fact_learner_course_status"),
50+
("mv", reporting, "fact_section_page_engagement"),
51+
("mv", reporting, "fact_section_problem_engagement"),
52+
("mv", reporting, "fact_section_video_engagement"),
53+
("mv", reporting, "fact_subsection_page_engagement"),
54+
("mv", reporting, "fact_subsection_problem_engagement"),
55+
("mv", reporting, "fact_subsection_video_engagement"),
56+
("mv", reporting, "int_videos_per_subsection"),
57+
("mv", reporting, "video_transcript_events"),
58+
("mv", xapi, "fact_instance_enrollments"),
6959
("mv", xapi, "fact_learner_last_course_visit"),
60+
("mv", xapi, "forum_events"),
7061
("mv", xapi, "responses"),
7162
("mv", xapi, "section_page_engagement"),
7263
("mv", xapi, "section_problem_engagement"),
7364
("mv", xapi, "section_video_engagement"),
7465
("mv", xapi, "subsection_page_engagement"),
7566
("mv", xapi, "subsection_problem_engagement"),
7667
("mv", xapi, "subsection_video_engagement"),
77-
("mv", xapi, "fact_instance_enrollments"),
78-
("mv", reporting, "int_videos_per_subsection"),
79-
("mv", reporting, "fact_section_video_engagement"),
80-
("mv", reporting, "fact_subsection_video_engagement"),
81-
("mv", reporting, "dim_learner_first_response"),
82-
("mv", reporting, "dim_learner_first_success_response"),
83-
("mv", reporting, "dim_learner_response_attempt"),
84-
("mv", reporting, "fact_section_problem_engagement"),
85-
("mv", reporting, "fact_subsection_problem_engagement"),
86-
("mv", reporting, "fact_section_page_engagement"),
87-
("mv", reporting, "fact_subsection_page_engagement"),
88-
("dictionary", event_sink, "course_names"),
89-
("dictionary", event_sink, "course_block_names"),
90-
("dictionary", event_sink, "course_tags"),
68+
("view", reporting, "dim_at_risk_learners"),
69+
("view", reporting, "dim_course_blocks_extended"),
70+
("view", reporting, "dim_user_pii"),
71+
("view", reporting, "fact_completions"),
72+
("view", reporting, "fact_enrollments"),
73+
("view", reporting, "fact_grades"),
74+
("view", reporting, "fact_learner_problem_summary"),
75+
("view", reporting, "fact_learner_response_attempts"),
76+
("view", reporting, "fact_navigation_dropoff"),
77+
("view", reporting, "fact_navigation"),
78+
("view", reporting, "fact_problem_engagement_per_subsection"),
79+
("view", reporting, "fact_problem_responses_extended"),
80+
("view", reporting, "fact_problem_responses"),
81+
("view", reporting, "fact_student_status"),
82+
("view", reporting, "fact_transcript_usage"),
83+
("view", reporting, "fact_video_plays"),
84+
("view", reporting, "fact_watched_video_duration"),
85+
("view", reporting, "fact_watched_video_segments"),
86+
("view", reporting, "int_pages_per_subsection"),
87+
("view", reporting, "int_problem_hints"),
88+
("view", reporting, "int_problem_results"),
89+
("view", reporting, "int_problems_per_subsection"),
90+
("view", reporting, "learner_summary"),
91+
("view", reporting, "most_recent_course_tags"),
92+
("view", reporting, "watched_video_duration"),
93+
("view", xapi, "fact_forum_interactions"),
9194
] %}
9295

9396
{% for model in models_to_drop %}

‎models/_docs/docs.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,14 @@ The "id" portion of the xAPI Verb for this event. See: https://xapi.com/statemen
116116
Number indicating which attempt this was
117117
{% enddocs %}
118118

119-
{% docs problem_number %}
119+
{% docs problem_location %}
120120
The section, subsection, unit, and part number of the problem block. In the format 1:2:3_1
121121
{% enddocs %}
122122

123+
{% docs problem_name_location %}
124+
The problem_location followed by the name of the problem
125+
{% enddocs %}
126+
123127
{% docs success %}
124128
Boolean indicating whether the responses were correct
125129
{% enddocs %}

‎models/courses/dim_course_blocks.sql‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,12 @@
1+
{{
2+
config(
3+
materialized="materialized_view",
4+
engine=get_engine("ReplacingMergeTree()"),
5+
primary_key="(org, course_key, block_id)",
6+
order_by="(org, course_key, block_id)",
7+
)
8+
}}
9+
110
with
211
dim_course_blocks as (
312
select
@@ -7,10 +16,12 @@ with
716
courses.course_run as course_run,
817
blocks.location as block_id,
918
blocks.block_name as block_name,
10-
{{ section_from_display("blocks.display_name_with_location") }}
11-
as section_number,
12-
{{ subsection_from_display("blocks.display_name_with_location") }}
13-
as subsection_number,
19+
concat(
20+
regexpExtract(blocks.display_name_with_location, '(\d+):'), ':0:0'
21+
) as section_number,
22+
concat(
23+
regexpExtract(blocks.display_name_with_location, '(.+):'), ':0'
24+
) as subsection_number,
1425
splitByString(' - ', blocks.display_name_with_location)[
1526
1
1627
] as hierarchy_location,

‎models/courses/dim_items_per_subsection.sql‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
{{
2+
config(
3+
materialized="materialized_view",
4+
engine=get_engine("ReplacingMergeTree()"),
5+
primary_key="(org, course_key, block_type, section_number, subsection_number)",
6+
order_by="(org, course_key, block_type, section_number, subsection_number)",
7+
)
8+
}}
9+
10+
select
11+
ips.org as org,
12+
ips.course_key as course_key,
13+
ips.section_number as section_number,
14+
section_blocks.display_name_with_location as section_with_name,
15+
ips.subsection_number as subsection_number,
16+
subsection_blocks.display_name_with_location as subsection_with_name,
17+
count(*) as item_count,
18+
subsection_blocks.block_id as subsection_block_id,
19+
section_blocks.block_id as section_block_id,
20+
subsection_blocks.course_order as subsection_course_order,
21+
case
22+
when ips.block_id like '%vertical+block%'
23+
then 'vertical+block'
24+
when ips.block_id like '%problem+block%'
25+
then 'problem+block'
26+
when ips.block_id like '%video+block%'
27+
then 'video+block'
28+
else 'none'
29+
end as block_type
30+
from {{ ref("dim_course_blocks") }} ips
31+
left join
32+
{{ ref("dim_course_blocks") }} section_blocks
33+
on (
34+
ips.section_number = section_blocks.hierarchy_location
35+
and ips.org = section_blocks.org
36+
and ips.course_key = section_blocks.course_key
37+
and section_blocks.block_id like '%@chapter+block@%'
38+
)
39+
left join
40+
{{ ref("dim_course_blocks") }} subsection_blocks
41+
on (
42+
ips.subsection_number = subsection_blocks.hierarchy_location
43+
and ips.org = subsection_blocks.org
44+
and ips.course_key = subsection_blocks.course_key
45+
and subsection_blocks.block_id like '%@sequential+block@%'
46+
)
47+
where block_type <> 'none'
48+
group by
49+
org,
50+
course_key,
51+
section_number,
52+
section_with_name,
53+
subsection_number,
54+
subsection_with_name,
55+
subsection_block_id,
56+
section_block_id,
57+
subsection_course_order,
58+
block_type

0 commit comments

Comments
 (0)