diff --git a/channel_integrations/moodle/__init__.py b/channel_integrations/moodle/__init__.py new file mode 100644 index 0000000..51eb20f --- /dev/null +++ b/channel_integrations/moodle/__init__.py @@ -0,0 +1,5 @@ +""" +The Moodle Integrated Channel package. +""" + +__version__ = "0.1.0" diff --git a/channel_integrations/moodle/admin/__init__.py b/channel_integrations/moodle/admin/__init__.py new file mode 100644 index 0000000..3525b93 --- /dev/null +++ b/channel_integrations/moodle/admin/__init__.py @@ -0,0 +1,110 @@ +""" +Django admin integration for configuring moodle app to communicate with Moodle systems. +""" + +from django_object_actions import DjangoObjectActions + +from django import forms +from django.contrib import admin, messages +from django.core.exceptions import ValidationError +from django.http import HttpResponseRedirect +from django.utils.translation import gettext_lazy as _ + +from channel_integrations.integrated_channel.admin import BaseLearnerDataTransmissionAuditAdmin +from channel_integrations.moodle.models import MoodleEnterpriseCustomerConfiguration, MoodleLearnerDataTransmissionAudit + + +class MoodleEnterpriseCustomerConfigurationForm(forms.ModelForm): + """ + Django admin form for MoodleEnterpriseCustomerConfiguration. + """ + class Meta: + model = MoodleEnterpriseCustomerConfiguration + fields = '__all__' + + def clean(self): + cleaned_data = super().clean() + cleaned_username = cleaned_data.get('decrypted_username') + cleaned_password = cleaned_data.get('decrypted_password') + cleaned_token = cleaned_data.get('decrypted_token') + if cleaned_token and (cleaned_username or cleaned_password): + raise ValidationError(_('Cannot set both a Username/Password and Token')) + if (cleaned_username and not cleaned_password) or (cleaned_password and not cleaned_username): + raise ValidationError(_('Must set both a Username and Password, not just one')) + + +@admin.register(MoodleEnterpriseCustomerConfiguration) +class MoodleEnterpriseCustomerConfigurationAdmin(DjangoObjectActions, admin.ModelAdmin): + """ + Django admin model for MoodleEnterpriseCustomerConfiguration. + """ + + raw_id_fields = ( + 'enterprise_customer', + ) + + form = MoodleEnterpriseCustomerConfigurationForm + change_actions = ('force_content_metadata_transmission',) + + @admin.action( + description="Force content metadata transmission for this Enterprise Customer" + ) + def force_content_metadata_transmission(self, request, obj): + """ + Updates the modified time of the customer record to retransmit courses metadata + and redirects to configuration view with success or error message. + """ + try: + obj.enterprise_customer.save() + messages.success( + request, + f'''The moodle enterprise customer content metadata + “” was updated successfully.''', + ) + except ValidationError: + messages.error( + request, + f'''The moodle enterprise customer content metadata + “” was not updated successfully.''', + ) + return HttpResponseRedirect( + "/admin/moodle/moodleenterprisecustomerconfiguration" + ) + force_content_metadata_transmission.label = "Force content metadata transmission" + + +@admin.register(MoodleLearnerDataTransmissionAudit) +class MoodleLearnerDataTransmissionAuditAdmin(BaseLearnerDataTransmissionAuditAdmin): + """ + Django admin model for MoodleLearnerDataTransmissionAudit. + """ + list_display = ( + "enterprise_course_enrollment_id", + "course_id", + "status", + "modified", + ) + + readonly_fields = ( + "moodle_user_email", + "progress_status", + "content_title", + "enterprise_customer_name", + "friendly_status_message", + "api_record", + ) + + search_fields = ( + "moodle_user_email", + "enterprise_course_enrollment_id", + "course_id", + "content_title", + "friendly_status_message" + ) + + list_per_page = 1000 + + class Meta: + model = MoodleLearnerDataTransmissionAudit diff --git a/channel_integrations/moodle/apps.py b/channel_integrations/moodle/apps.py new file mode 100644 index 0000000..8925b69 --- /dev/null +++ b/channel_integrations/moodle/apps.py @@ -0,0 +1,14 @@ +""" +Enterprise Integrated Channel Moodle Django application initialization. +""" + +from django.apps import AppConfig + + +class MoodleConfig(AppConfig): + """ + Configuration for the Enterprise Integrated Channel Moodle Django application. + """ + name = 'channel_integrations.moodle' + verbose_name = 'Enterprise Moodle Integration' + label = 'moodle_channel' diff --git a/channel_integrations/moodle/client.py b/channel_integrations/moodle/client.py new file mode 100644 index 0000000..769dc6e --- /dev/null +++ b/channel_integrations/moodle/client.py @@ -0,0 +1,548 @@ +""" +Client for connecting to Moodle. +""" + +import json +import logging +import time +from http import HTTPStatus +from urllib.parse import urlencode, urljoin + +import requests + +from django.apps import apps + +from channel_integrations.exceptions import ClientError +from channel_integrations.integrated_channel.client import IntegratedChannelApiClient +from channel_integrations.utils import generate_formatted_log, stringify_and_store_api_record + +LOGGER = logging.getLogger(__name__) + + +class MoodleClientError(ClientError): + """ + Indicate a problem when interacting with Moodle. + """ + + def __init__(self, message, status_code=500, moodle_error=None): + """Save the status code and message raised from the client.""" + self.status_code = status_code + self.message = message + self.moodle_error = moodle_error + super().__init__(message, status_code) + + +class MoodleResponse: + """ + Represents an HTTP response with status code and textual content. + """ + + def __init__(self, status_code, text): + """Save the status code and text of the response.""" + self.status_code = status_code + self.text = text + + +def moodle_request_wrapper(method): + """ + Wraps requests to Moodle's API in a token check. + Will obtain a new token if there isn't one. + """ + + def inner(self, *args, **kwargs): + if not self.token: + self.token = self._get_access_token() # pylint: disable=protected-access + response = method(self, *args, **kwargs) + try: + body = response.json() + except (AttributeError, ValueError) as error: + # Moodle spits back an entire HTML page if something is wrong in our URL format. + # This cannot be converted to JSON thus the above fails miserably. + # The above can fail with different errors depending on the format of the returned page. + # Moodle of course does not tell us what is wrong in any part of this HTML. + log_msg = (f'Moodle API task "{method.__name__}" ' + f'for enterprise_customer_uuid "{self.enterprise_configuration.enterprise_customer.uuid}" ' + f'failed due to unknown error with code "{response.status_code}".') + raise ClientError(log_msg, response.status_code) from error + if isinstance(body, list): + # On course creation (and ONLY course creation) success, + # Moodle returns a list of JSON objects, because of course it does. + # Otherwise, it fails instantly and returns actual JSON. + return response + if isinstance(body, int): + # This only happens for grades AFAICT. Zero also doesn't necessarily mean success, + # but we have nothing else to go on + if body == 0: + if method.__name__ == "_wrapped_create_course_completion" and response.status_code == 200: + return MoodleResponse(status_code=200, text='') + return 200, '' + raise ClientError('Moodle API Grade Update failed with int code: {code}'.format(code=body), 500) + if isinstance(body, str): + # Grades + debug can sometimes produce lines with debug errors and also "0" + raise ClientError('Moodle API Grade Update failed with possible error: {body}'.format(body=body), 500) + error_code = body.get('errorcode') + warnings = body.get('warnings') + if error_code and error_code == 'invalidtoken': + self.token = self._get_access_token() # pylint: disable=protected-access + response = method(self, *args, **kwargs) + elif error_code: + raise MoodleClientError( + 'Moodle API Client Task "{method}" failed with error code ' + '"{code}" and message: "{msg}" '.format( + method=method.__name__, code=error_code, msg=body.get('message'), + ), + response.status_code, + error_code, + ) + elif warnings: + # More Moodle nonsense! + errors = [] + for warning in warnings: + if warning.get('message'): + errors.append(warning.get('message')) + raise ClientError( + 'Moodle API Client Task "{method}" failed with the following error codes: ' + '"{code}"'.format(method=method.__name__, code=errors) + ) + return response + return inner + + +class MoodleAPIClient(IntegratedChannelApiClient): + """ + Client for connecting to Moodle. + Transmits learner and course metadata. + + Required configuration to access Moodle: + - wsusername and wspassword: + - Web service user and password created in Moodle. Used to generate api tokens. + - Moodle base url. + - Customer's Moodle instance url. + - For local development just `http://localhost` (unless you needed a different port) + - Moodle service short name. + - Customer's Moodle service short name + """ + + MOODLE_API_PATH = 'webservice/rest/server.php' + + def __init__(self, enterprise_configuration): + """ + Instantiate a new client. + + Args: + enterprise_configuration (MoodleEnterpriseCustomerConfiguration): An enterprise customers's + configuration model for connecting with Moodle + """ + super().__init__(enterprise_configuration) + self.config = apps.get_app_config('moodle') + self.token = enterprise_configuration.decrypted_token or self._get_access_token() + self.api_url = urljoin(self.enterprise_configuration.moodle_base_url, self.MOODLE_API_PATH) + self.IntegratedChannelAPIRequestLogs = apps.get_model( + "integrated_channel", "IntegratedChannelAPIRequestLogs" + ) + + def _post(self, additional_params): + """ + Compile common params and run request's post function + """ + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + } + params = { + 'wstoken': self.token, + 'moodlewsrestformat': 'json', + } + params.update(additional_params) + + start_time = time.time() + response = requests.post( + url=self.api_url, + data=params, + headers=headers + ) + duration_seconds = time.time() - start_time + stringify_and_store_api_record( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + enterprise_customer_configuration_id=self.enterprise_configuration.id, + endpoint=self.api_url, + data=params, + time_taken=duration_seconds, + status_code=response.status_code, + response_body=response.text, + channel_name=self.enterprise_configuration.channel_code() + ) + + return response + + @moodle_request_wrapper + def _wrapped_post(self, additional_params): + """ + A version of _post which handles error cases, useful + for when the caller wants to examine errors + """ + return self._post(additional_params) + + def _get_access_token(self): + """ + Obtains a new access token from Moodle using username and password. + """ + querystring = { + 'service': self.enterprise_configuration.service_short_name + } + + url = urljoin(self.enterprise_configuration.moodle_base_url, 'login/token.php') + complete_url = "{}?{}".format(url, urlencode(querystring)) + start_time = time.time() + data = { + "username": self.enterprise_configuration.decrypted_username, + "password": self.enterprise_configuration.decrypted_password, + } + response = requests.post( + url, + params=querystring, + headers={ + 'Content-Type': 'application/x-www-form-urlencoded', + }, + data=data, + ) + duration_seconds = time.time() - start_time + stringify_and_store_api_record( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + enterprise_customer_configuration_id=self.enterprise_configuration.id, + endpoint=complete_url, + data=data, + time_taken=duration_seconds, + status_code=response.status_code, + response_body=response.text, + channel_name=self.enterprise_configuration.channel_code() + ) + + try: + data = response.json() + token = data['token'] + return token + except (KeyError, ValueError) as error: + raise ClientError( + "Failed to post access token. Received message={} from Moodle".format(response.text), + response.status_code + ) from error + + @moodle_request_wrapper + def _get_enrolled_users(self, course_id): + """ + Helper method to make a request for all user enrollments under a Moodle course ID. + """ + params = { + 'wstoken': self.token, + 'wsfunction': 'core_enrol_get_enrolled_users', + 'courseid': course_id, + 'moodlewsrestformat': 'json' + } + return self._post(params) + + def get_creds_of_user_in_course(self, course_id, user_email): + """ + Sort through a list of users in a Moodle course and find the ID matching a student's email. + """ + response = self._get_enrolled_users(course_id) + parsed_response = response.json() + user_id = None + + if isinstance(parsed_response, list): + for enrollment in parsed_response: + if enrollment.get('email') == user_email: + user_id = enrollment.get('id') + break + if not user_id: + raise ClientError( + "MoodleAPIClient request failed: 404 User enrollment not found under user={} in course={}.".format( + user_email, + course_id + ), + HTTPStatus.NOT_FOUND.value + ) + return user_id + + @moodle_request_wrapper + def _get_course_contents(self, course_id): + """ + Retrieve the metadata of a Moodle course by ID. + """ + params = { + 'wstoken': self.token, + 'wsfunction': 'core_course_get_contents', + 'courseid': course_id, + 'moodlewsrestformat': 'json' + } + complete_url = "{}?{}".format(self.api_url, urlencode(params)) + start_time = time.time() + response = requests.get( + self.api_url, + params=params + ) + duration_seconds = time.time() - start_time + self.IntegratedChannelAPIRequestLogs.store_api_call( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + enterprise_customer_configuration_id=self.enterprise_configuration.id, + endpoint=complete_url, + payload='', + time_taken=duration_seconds, + status_code=response.status_code, + response_body=response.text, + channel_name=self.enterprise_configuration.channel_code() + ) + return response + + def get_course_final_grade_module(self, course_id): + """ + Sort through a Moodle course's components for the specific shell assignment designated + to be the edX integrated Final Grade. This is currently done by module name. + + Returns: + - course_module_id (int): The ID of the shell assignment + - module_name (string): The string name of the module. Required for sending a grade update request. + """ + response = self._get_course_contents(course_id) + course_module_id = None + module_name = None + if isinstance(response.json(), list): + for course in response.json(): + if course.get('name') == 'General': + modules = course.get('modules') + for module in modules: + if module.get('name') == self.enterprise_configuration.grade_assignment_name: + course_module_id = module.get('id') + module_name = module.get('modname') + + if not course_module_id: + raise ClientError( + 'MoodleAPIClient request failed: 404 Completion course module not found in Moodle.' + ' The enterprise customer needs to create an activity within the course with the name ' + '"(edX integration) Final Grade"', + HTTPStatus.NOT_FOUND.value + ) + return course_module_id, module_name + + @moodle_request_wrapper + def _get_courses(self, key): + """ + Gets courses from Moodle by key (because we cannot update/delete without it). + """ + params = { + 'wstoken': self.token, + 'wsfunction': 'core_course_get_courses_by_field', + 'field': 'idnumber', + 'value': key, + 'moodlewsrestformat': 'json' + } + complete_url = "{}?{}".format(self.api_url, urlencode(params)) + start_time = time.time() + response = requests.get( + self.api_url, + params=params + ) + duration_seconds = time.time() - start_time + self.IntegratedChannelAPIRequestLogs.store_api_call( + enterprise_customer=self.enterprise_configuration.enterprise_customer, + enterprise_customer_configuration_id=self.enterprise_configuration.id, + endpoint=complete_url, + payload='', + time_taken=duration_seconds, + status_code=response.status_code, + response_body=response.text, + channel_name=self.enterprise_configuration.channel_code() + ) + return response + + def _get_course_id(self, key): + """ + Obtain course from Moodle by course key and parse out the id. No exception thrown. + """ + response = self._get_courses(key) + parsed_response = json.loads(response.text) + if not parsed_response.get('courses'): + return None + else: + return parsed_response['courses'][0]['id'] + + def get_course_id(self, key): + """ + Obtain course from Moodle by course key and parse out the id. Raise on not found. + """ + course_id = self._get_course_id(key) + if not course_id: + raise ClientError( + 'MoodleAPIClient request failed: 404 Course key ' + '"{}" not found in Moodle.'.format(key), + HTTPStatus.NOT_FOUND.value + ) + return course_id + + @moodle_request_wrapper + def _wrapped_create_course_completion(self, user_id, payload): + """ + Wrapped method to request and use Moodle course and user information in order + to post a final course grade for the user. + """ + completion_data = json.loads(payload) + + course_id = self.get_course_id(completion_data['courseID']) + course_module_id, module_name = self.get_course_final_grade_module(course_id) + moodle_user_id = self.get_creds_of_user_in_course(course_id, user_id) + + params = { + 'wsfunction': 'core_grades_update_grades', + 'source': module_name, + 'courseid': course_id, + 'component': 'mod_assign', + 'activityid': course_module_id, + 'itemnumber': 0, + 'grades[0][studentid]': moodle_user_id, + # The grade is exported as a decimal between [0-1] + 'grades[0][grade]': completion_data['grade'] * self.enterprise_configuration.grade_scale + } + + response = self._post(params) + + if hasattr(response, 'status_code'): + status_code = response.status_code + else: + status_code = None + + if hasattr(response, 'text'): + text = response.text + else: + text = None + + if hasattr(response, 'headers'): + headers = response.headers + else: + headers = None + + LOGGER.info( + 'Learner Data Transmission' + f'for course={completion_data["courseID"]} with data ' + f'source: {module_name}, ' + f'activityid: {course_module_id}, ' + f'grades[0][studentid]: {moodle_user_id}, ' + f'grades[0][grade]: {completion_data["grade"] * self.enterprise_configuration.grade_scale} ' + f' with response: {response} ' + f'Status Code: {status_code}, ' + f'Text: {text}, ' + f'Headers: {headers}, ' + ) + + return response + + def create_content_metadata(self, serialized_data): + """ + The below assumes the data is dict/object. + Format should look like: + { + courses[0][shortname]: 'value', + courses[0][fullname]: 'value', + [...] + courses[1][shortname]: 'value', + courses[1][fullname]: 'value', + [...] + } + when sending 1 course and its a dupe, treat as success. + when sending N courses and a dupe exists, throw exception since + there is no easy way to retry with just the non-dupes. + """ + # check to see if more than 1 course is being passed + more_than_one_course = serialized_data.get('courses[1][shortname]') + serialized_data['wsfunction'] = 'core_course_create_courses' + try: + moodle_course_id = self._get_course_id(serialized_data['courses[0][idnumber]']) + # Course already exists but is hidden - make it visible + if moodle_course_id: + serialized_data['courses[0][visible]'] = 1 + return self.update_content_metadata(serialized_data) + else: # create a new course + response = self._wrapped_post(serialized_data) + except MoodleClientError as error: + # treat duplicate as successful, but only if its a single course + # set chunk size settings to 1 if youre seeing a lot of these errors + if error.moodle_error == 'shortnametaken' and not more_than_one_course: + return 200, "shortnametaken" + elif error.moodle_error == 'courseidnumbertaken' and not more_than_one_course: + return 200, "courseidnumbertaken" + else: + raise error + return response.status_code, response.text + + def update_content_metadata(self, serialized_data): + moodle_course_id = self._get_course_id(serialized_data['courses[0][idnumber]']) + # if we cannot find the course, lets create it + if moodle_course_id: + serialized_data['courses[0][id]'] = moodle_course_id + serialized_data['wsfunction'] = 'core_course_update_courses' + response = self._wrapped_post(serialized_data) + return response.status_code, response.text + else: + return self.create_content_metadata(serialized_data) + + def delete_content_metadata(self, serialized_data): + response = self._get_courses(serialized_data['courses[0][idnumber]']) + parsed_response = json.loads(response.text) + if not parsed_response.get('courses'): + LOGGER.info( + generate_formatted_log( + self.enterprise_configuration.channel_code(), + self.enterprise_configuration.enterprise_customer.uuid, + None, + None, + 'No course found while attempting to delete edX course: ' + f'{serialized_data["courses[0][idnumber]"]} from moodle.' + ) + ) + # Hacky way of getting around the request wrapper validation + rsp = requests.Response() + rsp._content = bytearray('{"result": "Course not found."}', 'utf-8') # pylint: disable=protected-access + return rsp + moodle_course_id = parsed_response['courses'][0]['id'] + serialized_data['wsfunction'] = 'core_course_update_courses' + serialized_data['courses[0][id]'] = moodle_course_id + response = self._wrapped_post(serialized_data) + return response.status_code, response.text + + def create_assessment_reporting(self, user_id, payload): + """ + Not implemented yet + """ + + def cleanup_duplicate_assignment_records(self, courses): + """ + Not implemented yet. + """ + LOGGER.error( + generate_formatted_log( + self.enterprise_configuration.channel_code(), + self.enterprise_configuration.enterprise_customer.uuid, + None, + None, + "Moodle integrated channel does not yet support assignment deduplication." + ) + ) + + def create_course_completion(self, user_id, payload): + """Send course completion data to Moodle""" + # The base integrated channels transmitter expects a tuple of (code, body), + # but we need to wrap the requests + resp = self._wrapped_create_course_completion(user_id, payload) + completion_data = json.loads(payload) + LOGGER.info( + generate_formatted_log( + channel_name=self.enterprise_configuration.channel_code(), + enterprise_customer_uuid=self.enterprise_configuration.enterprise_customer.uuid, + course_or_course_run_key=completion_data['courseID'], + plugin_configuration_id=self.enterprise_configuration.id, + message=f'Response for Moodle Create Course Completion Request response: {resp} ' + ) + ) + return resp.status_code, resp.text + + @moodle_request_wrapper + def delete_course_completion(self, user_id, payload): + pass diff --git a/channel_integrations/moodle/exporters/__init__.py b/channel_integrations/moodle/exporters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel_integrations/moodle/exporters/content_metadata.py b/channel_integrations/moodle/exporters/content_metadata.py new file mode 100644 index 0000000..3ee3517 --- /dev/null +++ b/channel_integrations/moodle/exporters/content_metadata.py @@ -0,0 +1,123 @@ +""" +Content metadata exporter for Moodle +""" +from datetime import timezone +from logging import getLogger + +from dateutil.parser import parse + +from channel_integrations.integrated_channel.exporters.content_metadata import ContentMetadataExporter + +LOGGER = getLogger(__name__) + + +class MoodleContentMetadataExporter(ContentMetadataExporter): + """ + Moodle implementation of ContentMetadataExporter. + """ + DATA_TRANSFORM_MAPPING = { + 'fullname': 'title', + 'shortname': 'shortname', + 'idnumber': 'key', + 'summary': 'description', + 'startdate': 'start', + 'enddate': 'end', + 'categoryid': 'categoryid', + } + + LONG_STRING_LIMIT = 1700 # Actual maximum value we can support for any individual course + SKIP_KEY_IF_NONE = True + + def transform_shortname(self, content_metadata_item): + """ + We're prefixing Title to the key to make shortname a little nicer in Moodle's UI. + But because we use key elsewhere, I mapped this to itself + so it doesn't override all "key" references + """ + return '{} ({})'.format( + content_metadata_item.get('title'), + content_metadata_item.get('key') + ) + + def transform_title(self, content_metadata_item): + """ + Returns the course title with all organizations (partners) appended in parantheses + Returned format is: courseTitle - via edX.org (Partner) + """ + formatted_orgs = [] + for org in content_metadata_item.get('organizations', []): + split_org = org.partition(': ') # results in: ['first_part', 'delim', 'latter part'] + if split_org[2] == '': # Occurs when the delimiter isn't present in the string. + formatted_orgs.append(split_org[0]) # Returns the original string + else: + formatted_orgs.append(split_org[2]) + if not formatted_orgs: + final_orgs = '' + else: + final_orgs = ' ({})'.format(', '.join(formatted_orgs)) + + edx_formatted_title = '{} - via edX.org'.format(content_metadata_item.get('title')) + return '{}{}'.format( + edx_formatted_title, + final_orgs + ) + + def transform_description(self, content_metadata_item): + """ + Return the course description and enrollment url as Moodle' syllabus body attribute. + This will display in the Syllabus tab in Moodle. + """ + enrollment_url = content_metadata_item.get('enrollment_url', None) + base_description = 'Go to edX course page
'.format( + enrollment_url=enrollment_url) + full_description = content_metadata_item.get('full_description') or None + short_description = content_metadata_item.get('short_description') or None + if full_description and len(full_description + enrollment_url) <= self.LONG_STRING_LIMIT: + description = "{base_description}{full_description}".format( + base_description=base_description, + full_description=full_description + ) + elif short_description and len(short_description + enrollment_url) <= self.LONG_STRING_LIMIT: + short_description = content_metadata_item.get('short_description') + description = "{base_description}{short_description}".format( + base_description=base_description, short_description=short_description + ) + else: + description = "{base_description}{title}".format( + base_description=base_description, + title=content_metadata_item.get('title', '') + ) + + return description + + def transform_categoryid(self, content_metadata_item): # pylint: disable=unused-argument + """ + Returns the Moodle category id configured in the model. + ID 1 is Miscellaneous and is the default/basic category. + """ + return self.enterprise_configuration.category_id or 1 + + def transform_start(self, content_metadata_item): + """ + Converts start from ISO date string to int (required for Moodle's "startdate" field) + """ + start_date = content_metadata_item.get('start', None) + if start_date: + return int(parse(start_date).replace(tzinfo=timezone.utc).timestamp()) + return None + + def transform_end(self, content_metadata_item): + """ + Converts end from ISO date string to int (required for Moodle's "enddate" field) + """ + end_date = content_metadata_item.get('end', None) + if end_date: + return int(parse(end_date).replace(tzinfo=timezone.utc).timestamp()) + return None + + def _apply_delete_transformation(self, metadata): + """ + Specific transformations required for "deleting/hiding" a course on a Moodle. + """ + metadata['visible'] = 0 # hide a course on moodle - instead of true delete + return metadata diff --git a/channel_integrations/moodle/exporters/learner_data.py b/channel_integrations/moodle/exporters/learner_data.py new file mode 100644 index 0000000..5a0c518 --- /dev/null +++ b/channel_integrations/moodle/exporters/learner_data.py @@ -0,0 +1,80 @@ +""" +Learner data exporter for Enterprise Integrated Channel Moodle. +""" + +from logging import getLogger + +from django.apps import apps + +from channel_integrations.catalog_service_utils import get_course_id_for_enrollment +from channel_integrations.integrated_channel.exporters.learner_data import LearnerExporter +from channel_integrations.utils import generate_formatted_log, parse_datetime_to_epoch_millis + +LOGGER = getLogger(__name__) + + +class MoodleLearnerExporter(LearnerExporter): + """ + Class to provide a Moodle learner data transmission audit prepared for serialization. + """ + + def get_learner_data_records( + self, + enterprise_enrollment, + completed_date=None, + content_title=None, + progress_status=None, + course_completed=False, + **kwargs + ): # pylint: disable=arguments-differ + """ + Return a MoodleLearnerDataTransmissionAudit with the given enrollment and course completion data. + If no remote ID can be found, return None. + """ + enterprise_customer_user = enterprise_enrollment.enterprise_customer_user + moodle_completed_timestamp = None + if completed_date is not None: + moodle_completed_timestamp = parse_datetime_to_epoch_millis(completed_date) + + if enterprise_customer_user.user_email is None: + LOGGER.debug(generate_formatted_log( + 'moodle', + enterprise_customer_user.enterprise_customer.uuid, + enterprise_customer_user.user_id, + None, + ('get_learner_data_records finished. No learner data was sent for this LMS User Id because ' + 'Moodle User ID not found for [{name}]'.format( + name=enterprise_customer_user.enterprise_customer.name + )))) + return None + + MoodleLearnerDataTransmissionAudit = apps.get_model( + 'moodle', + 'MoodleLearnerDataTransmissionAudit' + ) + + percent_grade = kwargs.get('grade_percent', None) + course_id = get_course_id_for_enrollment(enterprise_enrollment) + # We only want to send one record per enrollment and course, so we check if one exists first. + learner_transmission_record = MoodleLearnerDataTransmissionAudit.objects.filter( + enterprise_course_enrollment_id=enterprise_enrollment.id, + course_id=course_id, + ).first() + if learner_transmission_record is None: + learner_transmission_record = MoodleLearnerDataTransmissionAudit( + enterprise_course_enrollment_id=enterprise_enrollment.id, + moodle_user_email=enterprise_customer_user.user_email, + user_email=enterprise_customer_user.user_email, + course_id=get_course_id_for_enrollment(enterprise_enrollment), + course_completed=course_completed, + grade=percent_grade, + completed_timestamp=completed_date, + content_title=content_title, + progress_status=progress_status, + moodle_completed_timestamp=moodle_completed_timestamp, + enterprise_customer_uuid=enterprise_customer_user.enterprise_customer.uuid, + plugin_configuration_id=self.enterprise_configuration.id, + ) + # We return one record here, with the course key, that was sent to the integrated channel. + # TODO: this shouldn't be necessary anymore and eventually phased out as part of tech debt + return [learner_transmission_record] diff --git a/channel_integrations/moodle/migrations/0001_initial.py b/channel_integrations/moodle/migrations/0001_initial.py new file mode 100644 index 0000000..5d9764e --- /dev/null +++ b/channel_integrations/moodle/migrations/0001_initial.py @@ -0,0 +1,92 @@ +# Generated by Django 4.2.18 on 2025-02-19 14:32 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import fernet_fields.fields +import model_utils.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('enterprise', '0228_alter_defaultenterpriseenrollmentrealization_realized_enrollment'), + ('channel_integration', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='MoodleLearnerDataTransmissionAudit', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('enterprise_customer_uuid', models.UUIDField(blank=True, null=True)), + ('user_email', models.CharField(blank=True, max_length=255, null=True)), + ('plugin_configuration_id', models.IntegerField(blank=True, null=True)), + ('enterprise_course_enrollment_id', models.IntegerField(blank=True, db_index=True, null=True)), + ('course_id', models.CharField(max_length=255)), + ('content_title', models.CharField(blank=True, default=None, max_length=255, null=True)), + ('course_completed', models.BooleanField(default=True)), + ('progress_status', models.CharField(blank=True, max_length=255)), + ('completed_timestamp', models.DateTimeField(blank=True, null=True)), + ('instructor_name', models.CharField(blank=True, max_length=255)), + ('grade', models.FloatField(blank=True, null=True)), + ('total_hours', models.FloatField(blank=True, null=True)), + ('subsection_id', models.CharField(blank=True, db_index=True, max_length=255, null=True)), + ('subsection_name', models.CharField(max_length=255, null=True)), + ('status', models.CharField(blank=True, max_length=100, null=True)), + ('error_message', models.TextField(blank=True, null=True)), + ('is_transmitted', models.BooleanField(default=False)), + ('friendly_status_message', models.CharField(blank=True, default=None, help_text='A user-friendly API response status message.', max_length=255, null=True)), + ('moodle_user_email', models.EmailField(help_text='The learner`s Moodle email. This must match the email on edX', max_length=255)), + ('moodle_completed_timestamp', models.CharField(blank=True, help_text='Represents the Moodle representation of a timestamp: yyyy-mm-dd, which is always 10 characters. Can be left unset for audit transmissions.', max_length=10, null=True)), + ('api_record', models.OneToOneField(blank=True, help_text='Data pertaining to the transmissions API request response.', null=True, on_delete=django.db.models.deletion.CASCADE, to='channel_integration.apiresponserecord')), + ], + ), + migrations.CreateModel( + name='MoodleEnterpriseCustomerConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ('display_name', models.CharField(blank=True, default='', help_text='A configuration nickname.', max_length=255)), + ('idp_id', models.CharField(blank=True, default='', help_text='If provided, will be used as IDP slug to locate remote id for learners', max_length=255)), + ('active', models.BooleanField(help_text='Is this configuration active?')), + ('dry_run_mode_enabled', models.BooleanField(default=False, help_text='Is this configuration in dry-run mode? (experimental)')), + ('show_course_price', models.BooleanField(default=False, help_text='Displays course price')), + ('channel_worker_username', models.CharField(blank=True, default='', help_text='Enterprise channel worker username to get JWT tokens for authenticating LMS APIs.', max_length=255)), + ('catalogs_to_transmit', models.TextField(blank=True, default='', help_text='A comma-separated list of catalog UUIDs to transmit. If blank, all customer catalogs will be transmitted. If there are overlapping courses in the customer catalogs, the overlapping course metadata will be selected from the newest catalog.')), + ('disable_learner_data_transmissions', models.BooleanField(default=False, help_text='When set to True, the configured customer will no longer receive learner data transmissions, both scheduled and signal based', verbose_name='Disable Learner Data Transmission')), + ('last_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Content or Learner data record sync attempt', null=True)), + ('last_content_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Content data record sync attempt', null=True)), + ('last_learner_sync_attempted_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent Learner data record sync attempt', null=True)), + ('last_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Content or Learner data record sync attempt', null=True)), + ('last_content_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Content data record sync attempt', null=True)), + ('last_learner_sync_errored_at', models.DateTimeField(blank=True, help_text='The DateTime of the most recent failure of a Learner data record sync attempt', null=True)), + ('last_modified_at', models.DateTimeField(auto_now=True, help_text='The DateTime of the last change made to this configuration.', null=True)), + ('moodle_base_url', models.CharField(blank=True, help_text='The base URL used for API requests to Moodle', max_length=255, verbose_name='Moodle Base URL')), + ('service_short_name', models.CharField(blank=True, help_text='The short name for the Moodle webservice.', max_length=255, verbose_name='Webservice Short Name')), + ('category_id', models.IntegerField(blank=True, help_text='The category ID for what edX courses should be associated with.', null=True, verbose_name='Category ID')), + ('decrypted_username', fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's username used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Username')), + ('decrypted_password', fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's password used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Password')), + ('decrypted_token', fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's token used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Token')), + ('transmission_chunk_size', models.IntegerField(default=1, help_text='The maximum number of data items to transmit to the integrated channel with each request.')), + ('grade_scale', models.IntegerField(default=100, help_text='The maximum grade points for the courses. Default: 100', verbose_name='Grade Scale')), + ('grade_assignment_name', models.CharField(default='(edX integration) Final Grade', help_text='The name for the grade assigment created for the grade integration.', max_length=255, verbose_name='Grade Assignment Name')), + ('enable_incomplete_progress_transmission', models.BooleanField(default=False, help_text='When set to True, the configured customer will receive learner data transmissions, for incomplete courses as well')), + ('enterprise_customer', models.ForeignKey(help_text='Enterprise Customer associated with the configuration.', on_delete=django.db.models.deletion.CASCADE, related_name='moodle_enterprisecustomerpluginconfiguration', to='enterprise.enterprisecustomer')), + ], + ), + migrations.AddConstraint( + model_name='moodlelearnerdatatransmissionaudit', + constraint=models.UniqueConstraint(fields=('enterprise_course_enrollment_id', 'course_id'), name='moodle_ch_unique_enrollment_course_id'), + ), + migrations.AlterIndexTogether( + name='moodlelearnerdatatransmissionaudit', + index_together={('enterprise_customer_uuid', 'plugin_configuration_id')}, + ), + ] diff --git a/channel_integrations/moodle/migrations/__init__.py b/channel_integrations/moodle/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel_integrations/moodle/models.py b/channel_integrations/moodle/models.py new file mode 100644 index 0000000..76eb48f --- /dev/null +++ b/channel_integrations/moodle/models.py @@ -0,0 +1,332 @@ +""" +Database models for Enterprise Integrated Channel Moodle. +""" + +import json +from logging import getLogger + +from fernet_fields import EncryptedCharField + +from django.db import models +from django.utils.encoding import force_bytes, force_str +from django.utils.translation import gettext_lazy as _ + +from enterprise.models import EnterpriseCustomer +from channel_integrations.integrated_channel.models import ( + EnterpriseCustomerPluginConfiguration, + LearnerDataTransmissionAudit, +) +from channel_integrations.moodle.exporters.content_metadata import MoodleContentMetadataExporter +from channel_integrations.moodle.exporters.learner_data import MoodleLearnerExporter +from channel_integrations.moodle.transmitters.content_metadata import MoodleContentMetadataTransmitter +from channel_integrations.moodle.transmitters.learner_data import MoodleLearnerTransmitter +from channel_integrations.utils import is_valid_url + +LOGGER = getLogger(__name__) + + +class MoodleEnterpriseCustomerConfiguration(EnterpriseCustomerPluginConfiguration): + """ + The Enterprise-specific configuration we need for integrating with Moodle. + + .. no_pii: + """ + + # TODO: Remove this override when we switch to enterprise-integrated-channels completely + enterprise_customer = models.ForeignKey( + EnterpriseCustomer, + related_name='moodle_enterprisecustomerpluginconfiguration', + blank=False, + null=False, + help_text=_("Enterprise Customer associated with the configuration."), + on_delete=models.deletion.CASCADE + ) + + moodle_base_url = models.CharField( + max_length=255, + blank=True, + verbose_name="Moodle Base URL", + help_text=_("The base URL used for API requests to Moodle") + ) + + service_short_name = models.CharField( + max_length=255, + blank=True, + verbose_name="Webservice Short Name", + help_text=_( + "The short name for the Moodle webservice." + ) + ) + + category_id = models.IntegerField( + blank=True, + null=True, + verbose_name="Category ID", + help_text=_( + "The category ID for what edX courses should be associated with." + ) + ) + + decrypted_username = EncryptedCharField( + max_length=255, + verbose_name="Encrypted Webservice Username", + blank=True, + help_text=_( + "The encrypted API user's username used to obtain new tokens." + " It will be encrypted when stored in the database." + ), + null=True, + ) + + @property + def encrypted_username(self): + """ + Return encrypted username as a string. + + The data is encrypted in the DB at rest, but is unencrypted in the app when retrieved through the + decrypted_username field. This method will encrypt the username again before sending. + """ + if self.decrypted_username: + return force_str( + self._meta.get_field('decrypted_username').fernet.encrypt( + force_bytes(self.decrypted_username) + ) + ) + return self.decrypted_username + + @encrypted_username.setter + def encrypted_username(self, value): + """ + Set the encrypted username. + """ + self.decrypted_username = value + + decrypted_password = EncryptedCharField( + max_length=255, + verbose_name="Encrypted Webservice Password", + blank=True, + help_text=_( + "The encrypted API user's password used to obtain new tokens." + " It will be encrypted when stored in the database." + ), + null=True, + ) + + @property + def encrypted_password(self): + """ + Return encrypted password as a string. + + The data is encrypted in the DB at rest, but is unencrypted in the app when retrieved through the + decrypted_password field. This method will encrypt the password again before sending. + """ + if self.decrypted_password: + return force_str( + self._meta.get_field('decrypted_password').fernet.encrypt( + force_bytes(self.decrypted_password) + ) + ) + return self.decrypted_password + + @encrypted_password.setter + def encrypted_password(self, value): + """ + Set the encrypted password. + """ + self.decrypted_password = value + + decrypted_token = EncryptedCharField( + max_length=255, + verbose_name="Encrypted Webservice Token", + blank=True, + help_text=_( + "The encrypted API user's token used to obtain new tokens." + " It will be encrypted when stored in the database." + ), + null=True, + ) + + @property + def encrypted_token(self): + """ + Return encrypted token as a string. + + The data is encrypted in the DB at rest, but is unencrypted in the app when retrieved through the + decrypted_token field. This method will encrypt the token again before sending. + """ + if self.decrypted_token: + return force_str( + self._meta.get_field('decrypted_token').fernet.encrypt( + force_bytes(self.decrypted_token) + ) + ) + return self.decrypted_token + + @encrypted_token.setter + def encrypted_token(self, value): + """ + Set the encrypted token. + """ + self.decrypted_token = value + + transmission_chunk_size = models.IntegerField( + default=1, + help_text=_("The maximum number of data items to transmit to the integrated channel with each request.") + ) + + grade_scale = models.IntegerField( + default=100, + verbose_name="Grade Scale", + help_text=_("The maximum grade points for the courses. Default: 100") + ) + + grade_assignment_name = models.CharField( + default="(edX integration) Final Grade", + max_length=255, + verbose_name="Grade Assignment Name", + help_text=_( + "The name for the grade assigment created for the grade integration." + ) + ) + + enable_incomplete_progress_transmission = models.BooleanField( + help_text=_("When set to True, the configured customer will receive learner data transmissions, for incomplete" + " courses as well"), + default=False, + ) + + class Meta: + app_label = 'moodle_channel' + + @property + def is_valid(self): + """ + Returns whether or not the configuration is valid and ready to be activated + + Args: + obj: The instance of MoodleEnterpriseCustomerConfiguration + being rendered with this admin form. + """ + missing_items = {'missing': []} + incorrect_items = {'incorrect': []} + if not self.moodle_base_url: + missing_items.get('missing').append('moodle_base_url') + if not self.decrypted_token and not (self.decrypted_username and self.decrypted_password): + missing_items.get('missing').append('token OR username and password') + if not self.service_short_name: + missing_items.get('missing').append('service_short_name') + if not is_valid_url(self.moodle_base_url): + incorrect_items.get('incorrect').append('moodle_base_url') + if len(self.display_name) > 20: + incorrect_items.get('incorrect').append('display_name') + return missing_items, incorrect_items + + def __str__(self): + """ + Return human-readable string representation. + """ + return "".format( + enterprise_name=self.enterprise_customer.name + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + @staticmethod + def channel_code(): + """ + Returns an capitalized identifier for this channel class, unique among subclasses. + """ + return 'MOODLE' + + def get_learner_data_exporter(self, user): + return MoodleLearnerExporter(user, self) + + def get_learner_data_transmitter(self): + return MoodleLearnerTransmitter(self) + + def get_content_metadata_exporter(self, user): + return MoodleContentMetadataExporter(user, self) + + def get_content_metadata_transmitter(self): + return MoodleContentMetadataTransmitter(self) + + +class MoodleLearnerDataTransmissionAudit(LearnerDataTransmissionAudit): + """ + The payload we send to Moodle at a given point in time for an enterprise course enrollment. + + """ + moodle_user_email = models.EmailField( + max_length=255, + blank=False, + null=False, + help_text='The learner`s Moodle email. This must match the email on edX' + ) + + moodle_completed_timestamp = models.CharField( + null=True, + blank=True, + max_length=10, + help_text=( + 'Represents the Moodle representation of a timestamp: yyyy-mm-dd, ' + 'which is always 10 characters. Can be left unset for audit transmissions.' + ) + ) + + class Meta: + app_label = 'moodle_channel' + constraints = [ + models.UniqueConstraint( + fields=['enterprise_course_enrollment_id', 'course_id'], + name='moodle_ch_unique_enrollment_course_id' + ) + ] + index_together = ['enterprise_customer_uuid', 'plugin_configuration_id'] + + def __str__(self): + """ + Return a human-readable string representation of the object. + """ + return ( + ''.format( + transmission_id=self.id, + enterprise_course_enrollment_id=self.enterprise_course_enrollment_id, + moodle_user_email=self.moodle_user_email, + course_id=self.course_id + ) + ) + + def __repr__(self): + """ + Return uniquely identifying string representation. + """ + return self.__str__() + + def serialize(self, *args, **kwargs): + """ + Return a JSON-serialized representation. + + Sort the keys so the result is consistent and testable. + + # TODO: When we refactor to use a serialization flow consistent with how course metadata + # is serialized, remove the serialization here and make the learner data exporter handle the work. + """ + return json.dumps(self._payload_data(), sort_keys=True) + + def _payload_data(self): + """ + Convert the audit record's fields into Moodle key/value pairs. + """ + return { + 'userID': self.moodle_user_email, + 'courseID': self.course_id, + 'courseCompleted': 'true' if self.course_completed else 'false', + 'completedTimestamp': self.moodle_completed_timestamp, + 'grade': self.grade, + 'totalHours': self.total_hours, + } diff --git a/channel_integrations/moodle/transmitters/__init__.py b/channel_integrations/moodle/transmitters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/channel_integrations/moodle/transmitters/content_metadata.py b/channel_integrations/moodle/transmitters/content_metadata.py new file mode 100644 index 0000000..9b81487 --- /dev/null +++ b/channel_integrations/moodle/transmitters/content_metadata.py @@ -0,0 +1,42 @@ +""" +Class for transmitting content metadata to Moodle. +""" +import logging + +from channel_integrations.integrated_channel.transmitters.content_metadata import ContentMetadataTransmitter +from channel_integrations.moodle.client import MoodleAPIClient + +LOGGER = logging.getLogger(__name__) + + +class MoodleContentMetadataTransmitter(ContentMetadataTransmitter): + """ + This transmitter transmits exported content metadata to Moodle. + """ + + def __init__(self, enterprise_configuration, client=MoodleAPIClient): + """ + Use the ``MoodleAPIClient`` for content metadata transmission to Moodle. + """ + super().__init__( + enterprise_configuration=enterprise_configuration, + client=client + ) + + def _prepare_items_for_transmission(self, channel_metadata_items): + """ + Takes items from the exporter and formats the keys in the way required for Moodle. + """ + items = {} + for _, item in enumerate(channel_metadata_items): + for key in item: + new_key = 'courses[0][{}]'.format(key) + items[new_key] = item[key] + return items + + def _serialize_items(self, channel_metadata_items): + """ + Overrides the base class _serialize_items method such that we return an object + instead of a binary string. + """ + return self._prepare_items_for_transmission(channel_metadata_items) diff --git a/channel_integrations/moodle/transmitters/learner_data.py b/channel_integrations/moodle/transmitters/learner_data.py new file mode 100644 index 0000000..fe66ba4 --- /dev/null +++ b/channel_integrations/moodle/transmitters/learner_data.py @@ -0,0 +1,34 @@ +""" +Class for transmitting learner data to Moodle. +""" + +from channel_integrations.integrated_channel.transmitters.learner_data import LearnerTransmitter +from channel_integrations.moodle.client import MoodleAPIClient + + +class MoodleLearnerTransmitter(LearnerTransmitter): + """ + This endpoint is intended to receive learner data routed from the integrated_channel app that is ready to be + sent to Moodle. + """ + + def __init__(self, enterprise_configuration, client=MoodleAPIClient): + """ + By default, use the ``MoodleAPIClient`` for learner data transmission to Moodle. + """ + super().__init__( + enterprise_configuration=enterprise_configuration, + client=client + ) + + def transmit(self, payload, **kwargs): + """ + Send a completion status call to Moodle using the client. + + Args: + payload: The learner data exporter for Moodle + """ + kwargs['app_label'] = 'moodle' + kwargs['model_name'] = 'MoodleLearnerDataTransmissionAudit' + kwargs['remote_user_id'] = 'moodle_user_email' + super().transmit(payload, **kwargs)