ntd: enrich mart tables with caltrans_district, handle ntd_id as string for joins

Author: charlie-costanzo

warehouse/models/mart/ntd_annual_reporting/dim_2022_agency_information.sql

@@ -3,9 +3,21 @@ WITH staging_agency_information AS (
     FROM {{ ref('stg_ntd__2022_agency_information') }}
 ),
 
-dim_2022_agency_information AS (
+dim_organizations AS (
+
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+
+),
+
+dim_2022_agency_information AS (
+    SELECT
+        staging_agency_information.*,
+        dim_organizations.caltrans_district
     FROM staging_agency_information
+    LEFT JOIN dim_organizations
+        ON staging_agency_information.ntd_id = dim_organizations.ntd_id_2022
 )
 
 SELECT
@@ -51,6 +63,7 @@ SELECT
     reported_by_ntd_id,
     density,
     state_parent_ntd_id,
+    caltrans_district,
     dt,
     execution_ts
 FROM dim_2022_agency_information

warehouse/models/mart/ntd_annual_reporting/dim_2023_agency_information.sql

@@ -3,9 +3,21 @@ WITH staging_agency_information AS (
     FROM {{ ref('stg_ntd__2023_agency_information') }}
 ),
 
-dim_2023_agency_information AS (
+dim_organizations AS (
+
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+
+),
+
+dim_2023_agency_information AS (
+    SELECT
+        staging_agency_information.*,
+        dim_organizations.caltrans_district
     FROM staging_agency_information
+    LEFT JOIN dim_organizations
+        ON staging_agency_information.ntd_id = dim_organizations.ntd_id
 )
 
 SELECT

warehouse/models/mart/ntd_annual_reporting/fct_2023_contractual_relationships.sql

@@ -3,9 +3,21 @@ WITH staging_contractual_relationships AS (
     FROM {{ ref('stg_ntd__2023_contractual_relationships') }}
 ),
 
-fct_2023_contractual_relationships AS (
+dim_organizations AS (
+
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+
+),
+
+fct_2023_contractual_relationships AS (
+    SELECT
+        staging_contractual_relationships.*,
+        dim_organizations.caltrans_district
     FROM staging_contractual_relationships
+    LEFT JOIN dim_organizations
+        ON staging_contractual_relationships.ntd_id = dim_organizations.ntd_id
 )
 
 SELECT

warehouse/models/mart/ntd_annual_reporting/fct_breakdowns.sql

@@ -3,9 +3,24 @@ WITH staging_breakdowns AS (
     FROM {{ ref('stg_ntd__breakdowns') }}
 ),
 
-fct_breakdowns AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_breakdowns AS (
+    SELECT
+        staging_breakdowns.*,
+        dim_organizations.caltrans_district
     FROM staging_breakdowns
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_breakdowns.report_year = 2022 THEN
+                staging_breakdowns.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_breakdowns.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -38,6 +53,7 @@ SELECT
     vehicle_passenger_car_miles_1,
     vehicle_passenger_car_miles_2,
     vehicle_passenger_car_revenue,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_breakdowns

warehouse/models/mart/ntd_annual_reporting/fct_breakdowns_by_agency.sql

@@ -3,9 +3,24 @@ WITH staging_breakdowns_by_agency AS (
     FROM {{ ref('stg_ntd__breakdowns_by_agency') }}
 ),
 
-fct_breakdowns_by_agency AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_breakdowns_by_agency AS (
+    SELECT
+        staging_breakdowns_by_agency.*,
+        dim_organizations.caltrans_district
     FROM staging_breakdowns_by_agency
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_breakdowns_by_agency.report_year = 2022 THEN
+                staging_breakdowns_by_agency.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_breakdowns_by_agency.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -33,6 +48,7 @@ SELECT
     sum_train_revenue_miles,
     sum_vehicle_passenger_car_miles,
     sum_vehicle_passenger_car_revenue,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_breakdowns_by_agency

warehouse/models/mart/ntd_annual_reporting/fct_capital_expenses_by_capital_use.sql

@@ -3,9 +3,24 @@ WITH staging_capital_expenses_by_capital_use AS (
     FROM {{ ref('stg_ntd__capital_expenses_by_capital_use') }}
 ),
 
-fct_capital_expenses_by_capital_use AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_capital_expenses_by_capital_use AS (
+    SELECT
+        staging_capital_expenses_by_capital_use.*,
+        dim_organizations.caltrans_district
     FROM staging_capital_expenses_by_capital_use
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_capital_expenses_by_capital_use.report_year = 2022 THEN
+                staging_capital_expenses_by_capital_use.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_capital_expenses_by_capital_use.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -47,6 +62,7 @@ SELECT
     typeofservicecd,
     uace_code,
     uza_name,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_capital_expenses_by_capital_use

warehouse/models/mart/ntd_annual_reporting/fct_capital_expenses_by_mode.sql

@@ -3,9 +3,24 @@ WITH staging_capital_expenses_by_mode AS (
     FROM {{ ref('stg_ntd__capital_expenses_by_mode') }}
 ),
 
-fct_capital_expenses_by_mode AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_capital_expenses_by_mode AS (
+    SELECT
+        staging_capital_expenses_by_mode.*,
+        dim_organizations.caltrans_district
     FROM staging_capital_expenses_by_mode
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_capital_expenses_by_mode.report_year = 2022 THEN
+                staging_capital_expenses_by_mode.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_capital_expenses_by_mode.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -43,6 +58,7 @@ SELECT
     sum_stations,
     sum_total,
     typeofservicecd,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_capital_expenses_by_mode

warehouse/models/mart/ntd_annual_reporting/fct_capital_expenses_for_existing_service.sql

@@ -3,9 +3,24 @@ WITH staging_capital_expenses_for_existing_service AS (
     FROM {{ ref('stg_ntd__capital_expenses_for_existing_service') }}
 ),
 
-fct_capital_expenses_for_existing_service AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_capital_expenses_for_existing_service AS (
+    SELECT
+        staging_capital_expenses_for_existing_service.*,
+        dim_organizations.caltrans_district
     FROM staging_capital_expenses_for_existing_service
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_capital_expenses_for_existing_service.report_year = 2022 THEN
+                staging_capital_expenses_for_existing_service.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_capital_expenses_for_existing_service.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -32,6 +47,7 @@ SELECT
     sum_reduced_reporter,
     sum_stations,
     sum_total,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_capital_expenses_for_existing_service

warehouse/models/mart/ntd_annual_reporting/fct_capital_expenses_for_expansion_of_service.sql

@@ -3,9 +3,24 @@ WITH staging_capital_expenses_for_expansion_of_service AS (
     FROM {{ ref('stg_ntd__capital_expenses_for_expansion_of_service') }}
 ),
 
-fct_capital_expenses_for_expansion_of_service AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_capital_expenses_for_expansion_of_service AS (
+    SELECT
+        staging_capital_expenses_for_expansion_of_service.*,
+        dim_organizations.caltrans_district
     FROM staging_capital_expenses_for_expansion_of_service
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_capital_expenses_for_expansion_of_service.report_year = 2022 THEN
+                staging_capital_expenses_for_expansion_of_service.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_capital_expenses_for_expansion_of_service.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -32,6 +47,7 @@ SELECT
     sum_reduced_reporter,
     sum_stations,
     sum_total,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_capital_expenses_for_expansion_of_service

warehouse/models/mart/ntd_annual_reporting/fct_employees_by_agency.sql

@@ -3,9 +3,24 @@ WITH staging_employees_by_agency AS (
     FROM {{ ref('stg_ntd__employees_by_agency') }}
 ),
 
-fct_employees_by_agency AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_employees_by_agency AS (
+    SELECT
+        staging_employees_by_agency.*,
+        dim_organizations.caltrans_district
     FROM staging_employees_by_agency
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_employees_by_agency.report_year = 2022 THEN
+                staging_employees_by_agency.max_ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_employees_by_agency.max_ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -35,6 +50,7 @@ SELECT
     total_employees,
     total_operating_hours,
     total_salaries,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_employees_by_agency

warehouse/models/mart/ntd_annual_reporting/fct_employees_by_mode.sql

@@ -3,9 +3,24 @@ WITH staging_employees_by_mode AS (
     FROM {{ ref('stg_ntd__employees_by_mode') }}
 ),
 
-fct_employees_by_mode AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_employees_by_mode AS (
+    SELECT
+        staging_employees_by_mode.*,
+        dim_organizations.caltrans_district
     FROM staging_employees_by_mode
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_employees_by_mode.report_year = 2022 THEN
+                staging_employees_by_mode.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_employees_by_mode.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -28,6 +43,7 @@ SELECT
     sum_total_employee_count,
     sum_total_hours,
     type_of_service,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_employees_by_mode

warehouse/models/mart/ntd_annual_reporting/fct_employees_by_mode_and_employee_type.sql

@@ -3,9 +3,24 @@ WITH staging_employees_by_mode_and_employee_type AS (
     FROM {{ ref('stg_ntd__employees_by_mode_and_employee_type') }}
 ),
 
-fct_employees_by_mode_and_employee_type AS (
+dim_organizations AS (
     SELECT *
+    FROM {{ ref('dim_organizations') }}
+    WHERE _is_current
+),
+
+fct_employees_by_mode_and_employee_type AS (
+    SELECT
+        staging_employees_by_mode_and_employee_type.*,
+        dim_organizations.caltrans_district
     FROM staging_employees_by_mode_and_employee_type
+    LEFT JOIN dim_organizations
+        ON CASE
+            WHEN staging_employees_by_mode_and_employee_type.report_year = 2022 THEN
+                staging_employees_by_mode_and_employee_type.ntd_id = dim_organizations.ntd_id_2022
+            ELSE
+                staging_employees_by_mode_and_employee_type.ntd_id = dim_organizations.ntd_id
+        END
 )
 
 SELECT
@@ -48,6 +63,7 @@ SELECT
     vehicle_operations_count_q,
     vehicle_operations_hours,
     vehicle_operations_hours_q,
+    caltrans_district,
     dt,
     execution_ts
 FROM fct_employees_by_mode_and_employee_type