diff --git a/backend/cms_plugins.py b/backend/cms_plugins.py index 50fabda..d938b24 100644 --- a/backend/cms_plugins.py +++ b/backend/cms_plugins.py @@ -1,7 +1,12 @@ from cms.plugin_base import CMSPluginBase from cms.plugin_pool import plugin_pool from django.utils.translation import gettext_lazy as _ -from .models import HydroShareResource,HydroShareResourceList,ZoteroBibliographyResource +from .models import ( + HydroShareResource, + HydroShareResourceList, + ZoteroBibliographyResource, + HydroLearnModulesList, +) import logging from hs_restclient import HydroShare, HydroShareAuthBasic import uuid @@ -14,12 +19,14 @@ logger = logging.getLogger(__name__) + @plugin_pool.register_plugin class HydroShareResourcePlugin(CMSPluginBase): model = HydroShareResource name = _("HydroShare Resource Plugin") render_template = "hydroshare_resource_template.html" cache = False + def render(self, context, instance, placeholder): context = super().render(context, instance, placeholder) return context @@ -31,6 +38,7 @@ class HydroShareResourceListPlugin(CMSPluginBase): name = _("HydroShare Resource List Plugin") render_template = "hydroshare_list_resources.html" cache = False + def render(self, context, instance, placeholder): create_hydroshare_resources(instance) # instance.updated_version = instance.updated_version + 1 @@ -38,6 +46,21 @@ def render(self, context, instance, placeholder): context = super().render(context, instance, placeholder) return context + +@plugin_pool.register_plugin +class HydroLearnPlugin(CMSPluginBase): + model = HydroLearnModulesList + name = _("HydroLearn Plugin") + render_template = "hydrolearn_list_modules.html" + cache = False + + def render(self, context, instance, placeholder): + instance.updated_version = instance.updated_version + 1 + instance.save(update_fields=["updated_version"]) + context = super().render(context, instance, placeholder) + return context + + @plugin_pool.register_plugin class ZoteroBibliographyResourcePlugin(CMSPluginBase): model = ZoteroBibliographyResource @@ -45,26 +68,24 @@ class ZoteroBibliographyResourcePlugin(CMSPluginBase): render_template = "zotero_bibliography.html" cache = False - #This is key in order to call the API every time the page renders - #The instance.save calls the pre_save signal which makes the call of the API + # This is key in order to call the API every time the page renders + # The instance.save calls the pre_save signal which makes the call of the API def render(self, context, instance, placeholder): instance.updated_version = instance.updated_version + 1 - instance.save(update_fields=['updated_version']) + instance.save(update_fields=["updated_version"]) context = super().render(context, instance, placeholder) return context - + def create_hydroshare_resources(instance): logger.warning(instance.updated_version) keywords = [] - json_resources = { - 'list_resources': [] - } + json_resources = {"list_resources": []} # logging.warning(instance.user,instance.password) if instance.tags: - keywords = instance.tags.split(',') + keywords = instance.tags.split(",") # logging.warning(keywords) - if instance.user != '' and instance.password != '': + if instance.user != "" and instance.password != "": auth = HydroShareAuthBasic(username=instance.user, password=instance.password) hs = HydroShare(auth=auth) else: @@ -73,63 +94,68 @@ def create_hydroshare_resources(instance): try: # let's call the resources resources_api = hs.resources(subject=keywords) - #how about "nwm_portal_app" for "Tool Resources that are part of the apps page, and how about "nwm_portal_data" for Data Resources? - resources_model = instance.resources.get('list_resources',[]) + # how about "nwm_portal_app" for "Tool Resources that are part of the apps page, and how about "nwm_portal_data" for Data Resources? + resources_model = instance.resources.get("list_resources", []) # logging.warning(resources_model) for resource_api in resources_api: # logging.warning(resource_api['resource_title']) - matching_resource_model = get_dict_with_attribute(resources_model, 'resource_id', resource_api['resource_id']) + matching_resource_model = get_dict_with_attribute( + resources_model, "resource_id", resource_api["resource_id"] + ) # logging.warning(matching_resource_model) - + # If resource found locally, then check last update date if matching_resource_model: - is_recent_date = get_most_recent_date(matching_resource_model['date_last_updated'],resource_api['date_last_updated']) + is_recent_date = get_most_recent_date( + matching_resource_model["date_last_updated"], + resource_api["date_last_updated"], + ) # logging.warning(is_recent_date) - if is_recent_date : # If the resource retrieved from api is more recent, then update resource + if ( + is_recent_date + ): # If the resource retrieved from api is more recent, then update resource # logging.warning("resource has a more recent version") - single_resource = update_resource(resource_api,hs,instance) + single_resource = update_resource(resource_api, hs, instance) # logging.warning(single_resource) - json_resources['list_resources'].append(single_resource) + json_resources["list_resources"].append(single_resource) instance.resources = json_resources - else: # resource is the same, then retrive the resource saved locally + else: # resource is the same, then retrive the resource saved locally # logging.warning("resource is the same") single_resource = matching_resource_model - json_resources['list_resources'].append(single_resource) + json_resources["list_resources"].append(single_resource) instance.resources = json_resources # If the resource is not here then create one else: # logging.warning(resource) # logging.warning("resource is new, creating now") - single_resource = update_resource(resource_api,hs,instance) - json_resources['list_resources'].append(single_resource) + single_resource = update_resource(resource_api, hs, instance) + json_resources["list_resources"].append(single_resource) instance.resources = json_resources logging.warning(json_resources) - instance.save(update_fields=['resources']) + instance.save(update_fields=["resources"]) except Exception as e: - instance.resources = { - "Error":[ - f'The following error: {e}' - ] - } + instance.resources = {"Error": [f"The following error: {e}"]} + def get_dict_with_attribute(list_of_dicts, attribute, value): # Loop through each dictionary in the list for dictionary in list_of_dicts: # logging.warning(dictionary) # if attribute in dictionary: - # logging.warning(dictionary[attribute]) + # logging.warning(dictionary[attribute]) # Check if the attribute exists in the dictionary and has the specified value if attribute in dictionary and dictionary[attribute] == value: return dictionary # Return the dictionary if found return None # Return None if not found in any dictionary + def get_most_recent_date(date_local_resource, date_api): # logging.warning(f'{date_local_resource} , {date_api}') @@ -145,48 +171,79 @@ def get_most_recent_date(date_local_resource, date_api): else: return False -def update_resource(resource,hs,instance): + +def update_resource(resource, hs, instance): # logging.warning(science_metadata_json) - single_resource={} - if resource["resource_type"] == 'ToolResource': - science_metadata_json = hs.getScienceMetadata(resource['resource_id']) + single_resource = {} + if resource["resource_type"] == "ToolResource": + science_metadata_json = hs.getScienceMetadata(resource["resource_id"]) # logging.warning(f'{science_metadata_json}') - image_url = science_metadata_json.get('app_icon',instance.placeholder_image).get('value',instance.placeholder_image) - web_site_url = '' if not science_metadata_json.get('app_home_page_url','') else science_metadata_json.get('app_home_page_url').get('value','') - github_url = '' if not science_metadata_json.get('source_code_url','') else science_metadata_json.get('source_code_url').get('value','') - help_page_url = '' if not science_metadata_json.get('help_page_url','') else science_metadata_json.get('help_page_url').get('value','') - if resource["resource_type"] == 'CompositeResource': - resource_scrapping = requests.get(resource['resource_url']) - image_url = instance.placeholder_image if not extract_value_by_name(resource_scrapping.content,"app_icon") else extract_value_by_name(resource_scrapping.content,"app_icon") - web_site_url = '' if not extract_value_by_name(resource_scrapping.content,"home_page_url") else extract_value_by_name(resource_scrapping.content,"home_page_url") - github_url = '' if not extract_value_by_name(resource_scrapping.content,"source_code_url") else extract_value_by_name(resource_scrapping.content,"source_code_url") - help_page_url = '' if not extract_value_by_name(resource_scrapping.content,"help_page_url") else extract_value_by_name(resource_scrapping.content,"help_page_url") - if image_url == '': + image_url = science_metadata_json.get( + "app_icon", instance.placeholder_image + ).get("value", instance.placeholder_image) + web_site_url = ( + "" + if not science_metadata_json.get("app_home_page_url", "") + else science_metadata_json.get("app_home_page_url").get("value", "") + ) + github_url = ( + "" + if not science_metadata_json.get("source_code_url", "") + else science_metadata_json.get("source_code_url").get("value", "") + ) + help_page_url = ( + "" + if not science_metadata_json.get("help_page_url", "") + else science_metadata_json.get("help_page_url").get("value", "") + ) + if resource["resource_type"] == "CompositeResource": + resource_scrapping = requests.get(resource["resource_url"]) + image_url = ( + instance.placeholder_image + if not extract_value_by_name(resource_scrapping.content, "app_icon") + else extract_value_by_name(resource_scrapping.content, "app_icon") + ) + web_site_url = ( + "" + if not extract_value_by_name(resource_scrapping.content, "home_page_url") + else extract_value_by_name(resource_scrapping.content, "home_page_url") + ) + github_url = ( + "" + if not extract_value_by_name(resource_scrapping.content, "source_code_url") + else extract_value_by_name(resource_scrapping.content, "source_code_url") + ) + help_page_url = ( + "" + if not extract_value_by_name(resource_scrapping.content, "help_page_url") + else extract_value_by_name(resource_scrapping.content, "help_page_url") + ) + if image_url == "": image_url = instance.placeholder_image - - single_resource={ - 'title':resource['resource_title'], - 'abstract':resource['abstract'], - 'github_url': github_url, - 'image': image_url, - 'web_site_url': web_site_url, - 'documentation_url': help_page_url, - 'unique_identifier': f'{uuid.uuid4()}', - 'resource_id':resource['resource_id'], - 'date_last_updated': resource['date_last_updated'] + + single_resource = { + "title": resource["resource_title"], + "abstract": resource["abstract"], + "github_url": github_url, + "image": image_url, + "web_site_url": web_site_url, + "documentation_url": help_page_url, + "unique_identifier": f"{uuid.uuid4()}", + "resource_id": resource["resource_id"], + "date_last_updated": resource["date_last_updated"], } return single_resource def extract_value_by_name(html, name): - soup = BeautifulSoup(html, 'html.parser') - rows = soup.select('#extraMetaTable tbody tr') + soup = BeautifulSoup(html, "html.parser") + rows = soup.select("#extraMetaTable tbody tr") for row in rows: - name_cell = row.select_one('td:first-child') - value_cell = row.select_one('td:nth-child(2)') + name_cell = row.select_one("td:first-child") + value_cell = row.select_one("td:nth-child(2)") if name_cell and value_cell and name_cell.get_text(strip=True) == name: return value_cell.get_text(strip=True) - return None \ No newline at end of file + return None diff --git a/backend/migrations/0029_hydrolearnmoduleslist.py b/backend/migrations/0029_hydrolearnmoduleslist.py new file mode 100644 index 0000000..1343f44 --- /dev/null +++ b/backend/migrations/0029_hydrolearnmoduleslist.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2 on 2024-03-04 18:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cms', '0022_auto_20180620_1551'), + ('backend', '0028_hydroshareresourcelist_placeholder_image'), + ] + + operations = [ + migrations.CreateModel( + name='HydroLearnModulesList', + fields=[ + ('cmsplugin_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='backend_hydrolearnmoduleslist', serialize=False, to='cms.cmsplugin')), + ('organization', models.CharField(blank=True, default='', max_length=200)), + ('placeholder_image', models.CharField(default='https://picsum.photos/200', max_length=200)), + ('updated_version', models.IntegerField(default=0, editable=False)), + ('modules', models.JSONField(default=dict, editable=False)), + ], + options={ + 'abstract': False, + }, + bases=('cms.cmsplugin',), + ), + ] diff --git a/backend/models.py b/backend/models.py index b980abd..605606e 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,8 +1,9 @@ from cms.models.pluginmodel import CMSPlugin -from django.core.validators import MaxValueValidator, MinValueValidator +from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from pyzotero import zotero from hs_restclient import HydroShare, HydroShareAuthBasic +import requests # from datetime import datetime @@ -14,53 +15,71 @@ logger = logging.getLogger(__name__) + class HydroShareResource(CMSPlugin): - title = models.CharField(max_length=200, default='resource title') - subtitle = models.CharField(max_length=200, default='resource subtitle') - image = models.CharField(max_length=200, default='https://picsum.photos/200') - width= models.PositiveIntegerField(default=200, validators=[MinValueValidator(150), MaxValueValidator(400)]) - height=models.PositiveIntegerField(default=200, validators=[MinValueValidator(150), MaxValueValidator(400)]) - description= models.TextField(default='resource description') - github_url=models.CharField(max_length=200, default='', blank=True) - documentation_url=models.CharField(max_length=200, default='', blank=True) - web_site_url=models.CharField(max_length=200, default='', blank=True) - unique_identifier=models.UUIDField(default=uuid.uuid4, editable=False) + title = models.CharField(max_length=200, default="resource title") + subtitle = models.CharField(max_length=200, default="resource subtitle") + image = models.CharField(max_length=200, default="https://picsum.photos/200") + width = models.PositiveIntegerField( + default=200, validators=[MinValueValidator(150), MaxValueValidator(400)] + ) + height = models.PositiveIntegerField( + default=200, validators=[MinValueValidator(150), MaxValueValidator(400)] + ) + description = models.TextField(default="resource description") + github_url = models.CharField(max_length=200, default="", blank=True) + documentation_url = models.CharField(max_length=200, default="", blank=True) + web_site_url = models.CharField(max_length=200, default="", blank=True) + unique_identifier = models.UUIDField(default=uuid.uuid4, editable=False) class HydroShareResourceList(CMSPlugin): - user = models.CharField(max_length=200, default='', blank=True) - password = models.CharField(max_length=200, default='', blank=True) - placeholder_image = models.CharField(max_length=200, default='https://picsum.photos/200') + user = models.CharField(max_length=200, default="", blank=True) + password = models.CharField(max_length=200, default="", blank=True) + placeholder_image = models.CharField( + max_length=200, default="https://picsum.photos/200" + ) # width= models.PositiveIntegerField(default=200, validators=[MinValueValidator(150), MaxValueValidator(400)]) # height=models.PositiveIntegerField(default=200, validators=[MinValueValidator(150), MaxValueValidator(400)]) - tags=models.CharField(max_length=200, default='') - updated_version=models.IntegerField(default=0, editable=False) - resources=models.JSONField(editable=False,default=dict) + tags = models.CharField(max_length=200, default="") + updated_version = models.IntegerField(default=0, editable=False) + resources = models.JSONField(editable=False, default=dict) + + +class HydroLearnModulesList(CMSPlugin): + organization = models.CharField(max_length=200, default="", blank=True) + placeholder_image = models.CharField( + max_length=200, default="https://picsum.photos/200" + ) + updated_version = models.IntegerField(default=0, editable=False) + modules = models.JSONField(editable=False, default=dict) - class ZoteroBibliographyResource(CMSPlugin): - api_key = models.CharField(max_length=200, default='') - library_type=models.CharField(max_length=200, default='') - library_id = models.CharField(max_length=200, default='') - collection_id = models.CharField(max_length=200, default='', blank=True) - style= models.CharField(max_length=200, default='apa') - html=models.JSONField(editable=False) - unique_identifier=models.UUIDField(default=uuid.uuid4, editable=False) - updated_version=models.IntegerField(default=0, editable=False) - link_of_library_or_collection = models.CharField(max_length=400, default='') - + api_key = models.CharField(max_length=200, default="") + library_type = models.CharField(max_length=200, default="") + library_id = models.CharField(max_length=200, default="") + collection_id = models.CharField(max_length=200, default="", blank=True) + style = models.CharField(max_length=200, default="apa") + html = models.JSONField(editable=False) + unique_identifier = models.UUIDField(default=uuid.uuid4, editable=False) + updated_version = models.IntegerField(default=0, editable=False) + link_of_library_or_collection = models.CharField(max_length=400, default="") + + @receiver(pre_save, sender=ZoteroBibliographyResource) def create_html_citations(sender, instance, *args, **kwargs): params = { - 'include': 'bib,data', - 'style': 'apa', - 'sort': 'date', - 'direction': 'desc', - 'linkwrap': 1 + "include": "bib,data", + "style": "apa", + "sort": "date", + "direction": "desc", + "linkwrap": 1, } try: - zot = zotero.Zotero(instance.library_id, instance.library_type, instance.api_key) + zot = zotero.Zotero( + instance.library_id, instance.library_type, instance.api_key + ) if instance.collection_id: items = zot.collection_items(instance.collection_id, **params) else: @@ -81,9 +100,56 @@ def create_html_citations(sender, instance, *args, **kwargs): instance.html = publications_by_year except Exception as e: - instance.html = { - "Error":[ - f'The following error: {e}' - ] - } + instance.html = {"Error": [f"The following error: {e}"]} + + +@receiver(post_save, sender=HydroLearnModulesList) +def fetch_hydrolearn_modules(sender, instance, *args, **kwargs): + modules_list = [] + logger.warning("Fetching HydroLearn modules") + try: + URL = "https://edx.hydrolearn.org" + client = requests.session() + client.get(URL) # sets cookie for CSRF + csrftoken = client.cookies.get("csrftoken", "") or client.cookies.get( + "csrf", "" + ) + courses_url = f"{URL}/search/course_discovery/" + + login_data = {"csrfmiddlewaretoken": csrftoken} + courses_response = client.post( + courses_url, data=login_data, headers={"Referer": courses_url} + ) + courses_list = courses_response.json()["results"] + + if instance.organization: + + def is_from_organization(course): + return course["data"]["org"] == instance.organization + courses_list = filter(is_from_organization, courses_list) + for course in courses_list: + course_data = course["data"] + course_dict = { + "course_title": course_data["content"]["display_name"], + "course_url": f"{URL}/courses/{course_data.get('course')}/about", + "course_image_url": ( + f'{URL}{course_data.get("image_url")}' + if course_data.get("image_url", "") != "" + else "" + ), + "course_organization": course_data.get("org", ""), + "course_code": course_data.get("number", ""), + "course_weekly_effort": course_data.get("effort", ""), + "course_description_content": course_data.get("content").get( + "short_description", "" + ), + } + logger.warning(course_dict) + + modules_list.append(course_dict) + + instance.modules = modules_list + except Exception as e: + logger.warning(f"Error fetching HydroLearn modules: {e}") + instance.modules = modules_list diff --git a/backend/templates/hydrolearn_list_modules.html b/backend/templates/hydrolearn_list_modules.html new file mode 100644 index 0000000..9559dfb --- /dev/null +++ b/backend/templates/hydrolearn_list_modules.html @@ -0,0 +1,47 @@ +{% load cms_tags sekizai_tags %} + +{% addtoblock "css" %} + +{% endaddtoblock %} +
+ +
+ + +
+
+ {% for module in instance.modules %} + {% include "single_hydrolearn_module.html" with module=module %} + {% endfor %} +
+
\ No newline at end of file diff --git a/backend/templates/single_hydrolearn_module.html b/backend/templates/single_hydrolearn_module.html new file mode 100644 index 0000000..c683808 --- /dev/null +++ b/backend/templates/single_hydrolearn_module.html @@ -0,0 +1,57 @@ +{% load cms_tags sekizai_tags %} + +{% addtoblock "js" %} + +{% endaddtoblock %} + +
+
+
+ {% if resource.course_url %} + + ... + + {% else %} + + ... + + {% endif %} +
+
+
+
+
{{ module.course_title }}
+
+ +
+
+

+ {% if module.course_organization %} + {{ module.course_organization }} + {% endif %} +

+
+
+

+ {% if module.course_code %} + {{ module.course_code }} + {% endif %} +

+
+
+ + {% if module.course_weekly_effort %} +

Effort: {{ module.course_weekly_effort }}

+ {% endif %} + +
+

+ {{ module.course_description_content }} +

+
+
+
+
+
+ +