Merge branch 'master' into chris/FAL-4180-sections-subsections-publish

Author: rpenido

.coveragerc

@@ -25,6 +25,7 @@ omit =
     common/djangoapps/*/migrations/*
     openedx/core/djangoapps/*/migrations/*
     openedx/core/djangoapps/debug/*
+    openedx/envs/*
     openedx/features/*/migrations/*
 
 concurrency=multiprocessing

.coveragerc-local

@@ -24,6 +24,7 @@ omit =
     common/djangoapps/*/migrations/*
     openedx/core/djangoapps/*/migrations/*
     openedx/core/djangoapps/debug/*
+    openedx/envs/*
     openedx/features/*/migrations/*
 
 concurrency=multiprocessing

.github/workflows/pylint-checks.yml

@@ -20,7 +20,7 @@ jobs:
           - module-name: openedx-1
             path: "openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/content_staging/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/djangoapps/course_live/"
           - module-name: openedx-2
-            path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
+            path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/envs/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
           - module-name: common
             path: "common"
           - module-name: cms

cms/djangoapps/contentstore/core/course_optimizer_provider.py

@@ -251,7 +251,7 @@ def _get_node_path(block):
 }
 
 
-def _create_dto_recursive(xblock_node, xblock_dictionary):
+def _create_dto_recursive(xblock_node, xblock_dictionary, parent_id=None):
     """
     Recursively build the Data Transfer Object by using
     the structure from the node tree and data from the dictionary.
@@ -264,7 +264,7 @@ def _create_dto_recursive(xblock_node, xblock_dictionary):
     xblock_children = []
 
     for xblock_id, node in xblock_node.items():
-        child_blocks = _create_dto_recursive(node, xblock_dictionary)
+        child_blocks = _create_dto_recursive(node, xblock_dictionary, parent_id=xblock_id)
         xblock_data = xblock_dictionary.get(xblock_id, {})
 
         xblock_entry = {
@@ -281,6 +281,13 @@ def _create_dto_recursive(xblock_node, xblock_dictionary):
             })
         else:   # Non-leaf node
             category = xblock_data.get('category', None)
+            # If parent and child has same IDs and level is 'sections', change it to 'subsections'
+            # And if parent and child has same IDs and level is 'subsections', change it to 'units'
+            if xblock_id == parent_id:
+                if category == "chapter":
+                    category = "sequential"
+                elif category == "sequential":
+                    category = "vertical"
             level = CATEGORY_TO_LEVEL_MAP.get(category, None)
             xblock_entry.update(child_blocks)
 

cms/djangoapps/contentstore/signals/handlers.py

@@ -131,6 +131,8 @@ def emit_catalog_info_changed_signal(course_key: CourseKey):
     if SEND_CATALOG_INFO_SIGNAL.is_enabled():
         timestamp, catalog_info = _create_catalog_data_for_signal(course_key)
         if catalog_info is not None:
+            # .. event_implemented_name: COURSE_CATALOG_INFO_CHANGED
+            # .. event_type: org.openedx.content_authoring.course.catalog_info.changed.v1
             COURSE_CATALOG_INFO_CHANGED.send_event(time=timestamp, catalog_info=catalog_info)
 
 

cms/djangoapps/contentstore/tasks.py

@@ -37,7 +37,7 @@
 from opaque_keys.edx.locator import LibraryLocator, LibraryContainerLocator
 from organizations.api import add_organization_course, ensure_organization
 from organizations.exceptions import InvalidOrganizationException
-from organizations.models import Organization, OrganizationCourse
+from organizations.models import Organization
 from path import Path as path
 from pytz import UTC
 from user_tasks.models import UserTaskArtifact, UserTaskStatus
@@ -98,6 +98,15 @@
 ALL_ALLOWED_XBLOCKS = frozenset(
     [entry_point.name for entry_point in entry_points(group="xblock.v1")]
 )
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/115.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Connection": "keep-alive",
+}
 
 
 class LinkState:
@@ -163,12 +172,6 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i
         # call edxval to attach videos to the rerun
         copy_course_videos(source_course_key, destination_course_key)
 
-        # Copy OrganizationCourse
-        organization_course = OrganizationCourse.objects.filter(course_id=source_course_key_string).first()
-
-        if organization_course:
-            clone_instance(organization_course, {'course_id': destination_course_key_string})
-
         # Copy RestrictedCourse
         restricted_course = RestrictedCourse.objects.filter(course_key=source_course_key).first()
 
@@ -178,7 +181,7 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i
             for country_access_rule in country_access_rules:
                 clone_instance(country_access_rule, {'restricted_course': new_restricted_course})
 
-        org_data = ensure_organization(source_course_key.org)
+        org_data = ensure_organization(destination_course_key.org)
         add_organization_course(org_data, destination_course_key)
         return "succeeded"
 
@@ -473,12 +476,12 @@ def sync_discussion_settings(course_key, user):
 
         if (
             ENABLE_NEW_STRUCTURE_DISCUSSIONS.is_enabled()
-            and not course.discussions_settings['provider_type'] == Provider.OPEN_EDX
+            and not course.discussions_settings.get('provider_type', None) == Provider.OPEN_EDX
+            and not course.discussions_settings.get('provider', None) == Provider.OPEN_EDX
         ):
             LOGGER.info(f"New structure is enabled, also updating {course_key} to use new provider")
             course.discussions_settings['enable_graded_units'] = False
             course.discussions_settings['unit_level_visibility'] = True
-            course.discussions_settings['provider'] = Provider.OPEN_EDX
             course.discussions_settings['provider_type'] = Provider.OPEN_EDX
             modulestore().update_item(course, user.id)
 
@@ -1261,7 +1264,7 @@ async def _validate_urls_access_in_batches(url_list, course_key, batch_size=100)
 
 async def _validate_batch(batch, course_key):
     """Validate a batch of URLs"""
-    async with aiohttp.ClientSession() as session:
+    async with aiohttp.ClientSession(headers=DEFAULT_HEADERS) as session:
         tasks = [_validate_url_access(session, url_data, course_key) for url_data in batch]
         batch_results = await asyncio.gather(*tasks)
         return batch_results
@@ -1286,6 +1289,7 @@ async def _validate_url_access(session, url_data, course_key):
         }
     """
     block_id, url = url_data
+    url = url.strip()  # Trim leading/trailing whitespace
     result = {'block_id': block_id, 'url': url}
     standardized_url = _convert_to_standard_url(url, course_key)
     try:

cms/djangoapps/contentstore/tests/test_tasks.py

@@ -189,6 +189,26 @@ def test_success(self):
             country=restricted_country
         )
 
+    def test_success_different_org(self):
+        """
+        The task should clone the OrganizationCourse with a different org.
+        """
+        old_course_key = self.course.id
+        new_course_key = CourseLocator(org='neworg', course=old_course_key.course, run='rerun')
+
+        old_course_id = str(old_course_key)
+        new_course_id = str(new_course_key)
+
+        organization = OrganizationFactory(short_name=old_course_key.org)
+        OrganizationCourse.objects.create(course_id=old_course_id, organization=organization)
+
+        # Run the task!
+        self._rerun_course(old_course_key, new_course_key)
+
+        # Verify the OrganizationCourse is cloned with a different org
+        self.assertEqual(OrganizationCourse.objects.count(), 2)
+        OrganizationCourse.objects.get(course_id=new_course_id, organization__short_name='neworg')
+
 
 @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
 class RegisterExamsTaskTestCase(CourseTestCase):  # pylint: disable=missing-class-docstring

cms/djangoapps/contentstore/utils.py

@@ -2314,6 +2314,8 @@ def send_course_update_notification(course_key, content, user):
         app_name="updates",
         audience_filters={},
     )
+    # .. event_implemented_name: COURSE_NOTIFICATION_REQUESTED
+    # .. event_type: org.openedx.learning.course.notification.requested.v1
     COURSE_NOTIFICATION_REQUESTED.send_event(course_notification_data=notification_data)
 
 

cms/djangoapps/contentstore/video_storage_handlers.py

@@ -724,7 +724,9 @@ def get_all_transcript_languages():
     third_party_transcription_languages.update(cielo_fidelity['PREMIUM']['languages'])
     third_party_transcription_languages.update(cielo_fidelity['PROFESSIONAL']['languages'])
 
-    all_languages_dict = dict(settings.ALL_LANGUAGES, **third_party_transcription_languages)
+    # combines ALL_LANGUAGES with additional languages that should be supported for transcripts
+    extended_all_languages = settings.ALL_LANGUAGES + settings.EXTENDED_VIDEO_TRANSCRIPT_LANGUAGES
+    all_languages_dict = dict(extended_all_languages, **third_party_transcription_languages)
     # Return combined system settings and 3rd party transcript languages.
     all_languages = []
     for key, value in sorted(all_languages_dict.items(), key=lambda k_v: k_v[1]):

cms/djangoapps/contentstore/views/component.py

@@ -494,6 +494,11 @@ def create_support_legend_dict():
             course_advanced_keys
         )
     if advanced_component_templates['templates']:
+        # Advanced component templates should be sorted alphabetically by display name.
+        advanced_component_templates['templates'] = sorted(
+            advanced_component_templates['templates'],
+            key=lambda x: x.get('display_name')
+        )
         component_templates.append(advanced_component_templates)
 
     return component_templates