From dac18717233286a7cbc203b842a792a06b7c578a Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 26 Dec 2024 17:15:08 -0500 Subject: [PATCH 01/10] feat: add integration tests --- .github/workflows/integration-test.yaml | 53 +++++++++++ integration_tests/integration_test.py | 113 ++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 .github/workflows/integration-test.yaml create mode 100644 integration_tests/integration_test.py diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 0000000..b294540 --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,53 @@ +name: Integration Test + +on: [push, pull_request] + +jobs: + edx-platform-integration-test: + name: Integration with Tutor + strategy: + matrix: + # Open edX Version: Sumac + tutor_version: ["<20.0.0"] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: mitx-grading-library + + - name: Adjust permissions to execute Tutor commands + run: | + chmod 777 . -R + shell: bash + + - name: Set Tutor environment variables + run: | + cat <> "$GITHUB_ENV" + LMS_HOST=local.edly.io + CMS_HOST=studio.local.edly.io + TUTOR_ROOT=$(pwd) + EOF + shell: bash + + - name: Install and prepare Tutor, Codejail and tests + run: | + pip install "tutor${{ matrix.tutor_version }}" + pip install git+https://github.com/edunext/tutor-contrib-codejail + tutor config save + tutor plugins enable codejail + tutor local do init --limit codejail + tutor mounts add cms:mitx-grading-library/integration_tests/integration_test.py:/openedx/edx-platform/integration_test.py + tutor local launch -I + shell: bash + + - name: Import MITx Demo Course + run: | + REPO_URL="https://github.com/${{ github.repository }}" + tutor local do importdemocourse -r $REPO_URL -d course -v ${{ github.ref_name }} + shell: bash + + - name: Run integration tests + run: | + tutor local run cms python3 integration_test.py + shell: bash diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py new file mode 100644 index 0000000..71949dd --- /dev/null +++ b/integration_tests/integration_test.py @@ -0,0 +1,113 @@ +""" +This script is used to test the integration of the mitx-graders library with the Open edX platform. +Running a code that uses the functions provided by the library using the safe_exec function, and in the MIT course context, +to be able to use the python_lib.zip that contains the library. +""" + +import os +import django +import logging +from xmodule.capa.safe_exec import safe_exec +from xmodule.util.sandboxing import SandboxService +from xmodule.contentstore.django import contentstore +from opaque_keys.edx.keys import CourseKey + +# Set up Django environment +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.test") +django.setup() + +log = logging.getLogger(__name__) + +# Define the code to be executed +all_code = """ +from mitxgraders import * + +StringGrader(answers="cat") + +ListGrader( + answers=['cat', 'dog', 'unicorn'], + subgraders=StringGrader() +) + +FormulaGrader(answers='0') + +IntegralGrader( + answers={ + 'lower': 'a', + 'upper': 'b', + 'integrand': 'x^2', + 'integration_variable': 'x' + }, + input_positions={ + 'integrand': 1, + 'lower': 2, + 'upper': 3 + }, + variables=['a', 'b'] +) + +IntervalGrader(answers=['(','1','2',']']) + +MatrixGrader( + answers='x*A*B*u + z*C^3*v/(u*C*v)', + variables=['A', 'B', 'C', 'u', 'v', 'z', 'x'], + sample_from={ + 'A': RealMatrices(shape=[2, 3]), + 'B': RealMatrices(shape=[3, 2]), + 'C': RealMatrices(shape=[2, 2]), + 'u': RealVectors(shape=[2]), + 'v': RealVectors(shape=[2]), + 'z': ComplexRectangle() + }, + identity_dim=2 +) + +SumGrader( + answers={ + 'lower': 'a', + 'upper': 'b', + 'summand': 'x^2', + 'summation_variable': 'x' + }, + input_positions={ + 'summand': 1, + 'lower': 2, + 'upper': 3 + }, + variables=['a', 'b'] +) + +""" + + +def execute_code(all_code, course_key_str): + course_key = CourseKey.from_string(course_key_str) + sandbox_service = SandboxService( + course_id=course_key, + contentstore=contentstore) + zip_lib = sandbox_service.get_python_lib_zip() + + extra_files = [] + python_path = [] + + if zip_lib is not None: + extra_files.append(("python_lib.zip", zip_lib)) + python_path.append("python_lib.zip") + + try: + safe_exec( + all_code, + globals_dict={}, + python_path=python_path, + extra_files=extra_files, + slug="integration-test", + limit_overrides_context=course_key_str, + unsafely=False, + ) + except Exception as err: + log.error("Error executing code: %s", err) + + +if __name__ == "__main__": + course_key_str = "course-v1:MITx+grading-library+course" + execute_code(all_code, course_key_str) From dcee7d81052041d0248f51911932b8af79380b31 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 26 Dec 2024 18:02:23 -0500 Subject: [PATCH 02/10] fix: fix the repo url and branch --- .github/workflows/integration-test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index b294540..c169664 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -1,6 +1,6 @@ name: Integration Test -on: [push, pull_request] +on: [pull_request] jobs: edx-platform-integration-test: @@ -43,8 +43,7 @@ jobs: - name: Import MITx Demo Course run: | - REPO_URL="https://github.com/${{ github.repository }}" - tutor local do importdemocourse -r $REPO_URL -d course -v ${{ github.ref_name }} + tutor local do importdemocourse -r ${{ github.event.pull_request.head.repo.clone_url }} -d course -v ${{ github.event.pull_request.head.ref }} shell: bash - name: Run integration tests From 14e3e77444abba0020d47a62dadacf881f5098d4 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 27 Dec 2024 17:04:29 -0500 Subject: [PATCH 03/10] fix: improve docs and remove try except to allow the fail --- integration_tests/integration_test.py | 31 ++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py index 71949dd..f137365 100644 --- a/integration_tests/integration_test.py +++ b/integration_tests/integration_test.py @@ -81,6 +81,16 @@ def execute_code(all_code, course_key_str): + """ + Executes the provided code in a sandboxed environment with the specified course context. + + Args: + all_code (str): The code to be executed. + course_key_str (str): The string representation of the course key. + + Returns: + None + """ course_key = CourseKey.from_string(course_key_str) sandbox_service = SandboxService( course_id=course_key, @@ -94,18 +104,15 @@ def execute_code(all_code, course_key_str): extra_files.append(("python_lib.zip", zip_lib)) python_path.append("python_lib.zip") - try: - safe_exec( - all_code, - globals_dict={}, - python_path=python_path, - extra_files=extra_files, - slug="integration-test", - limit_overrides_context=course_key_str, - unsafely=False, - ) - except Exception as err: - log.error("Error executing code: %s", err) + safe_exec( + all_code, + globals_dict={}, + python_path=python_path, + extra_files=extra_files, + slug="integration-test", + limit_overrides_context=course_key_str, + unsafely=False, + ) if __name__ == "__main__": From bf4a14f31d3a14bc95b8af6f5ef1e7d6007f77e9 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 27 Dec 2024 17:48:19 -0500 Subject: [PATCH 04/10] docs: improve test --- integration_tests/integration_test.py | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py index f137365..6ee7ac0 100644 --- a/integration_tests/integration_test.py +++ b/integration_tests/integration_test.py @@ -22,14 +22,39 @@ all_code = """ from mitxgraders import * +# Grading Classes + +## Single-input graders + StringGrader(answers="cat") +FormulaGrader(answers='0') +NumericalGrader() +MatrixGrader( + answers='x*A*B*u + z*C^3*v/(u*C*v)', + variables=['A', 'B', 'C', 'u', 'v', 'z', 'x'], + sample_from={ + 'A': RealMatrices(shape=[2, 3]), + 'B': RealMatrices(shape=[3, 2]), + 'C': RealMatrices(shape=[2, 2]), + 'u': RealVectors(shape=[2]), + 'v': RealVectors(shape=[2]), + 'z': ComplexRectangle() + }, + identity_dim=2 +) +SingleListGrader( + answers=['cat', 'dog', 'unicorn'], + subgrader=StringGrader() +) + +## Multi-input graders ListGrader( answers=['cat', 'dog', 'unicorn'], subgraders=StringGrader() ) -FormulaGrader(answers='0') +## Specialized graders IntegralGrader( answers={ @@ -48,20 +73,6 @@ IntervalGrader(answers=['(','1','2',']']) -MatrixGrader( - answers='x*A*B*u + z*C^3*v/(u*C*v)', - variables=['A', 'B', 'C', 'u', 'v', 'z', 'x'], - sample_from={ - 'A': RealMatrices(shape=[2, 3]), - 'B': RealMatrices(shape=[3, 2]), - 'C': RealMatrices(shape=[2, 2]), - 'u': RealVectors(shape=[2]), - 'v': RealVectors(shape=[2]), - 'z': ComplexRectangle() - }, - identity_dim=2 -) - SumGrader( answers={ 'lower': 'a', From 5dd674854ba51d68a1925c89bbd301ac71d98832 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 30 Dec 2024 19:40:50 -0500 Subject: [PATCH 05/10] refactor: improve the code with feedback --- .github/workflows/integration-test.yaml | 21 ++++++++++++++------ integration_tests/integration_test.py | 26 +++++++++++++++---------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index c169664..0c6aeb4 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -27,18 +27,27 @@ jobs: LMS_HOST=local.edly.io CMS_HOST=studio.local.edly.io TUTOR_ROOT=$(pwd) + COURSE_KEY=course-v1:MITx+grading-library+course EOF shell: bash - - name: Install and prepare Tutor, Codejail and tests + - name: Install Tutor + run: pip install "tutor${{ matrix.tutor_version }}" + shell: bash + + - name: Install, enable and initialize Tutor Codejail Plugin run: | - pip install "tutor${{ matrix.tutor_version }}" pip install git+https://github.com/edunext/tutor-contrib-codejail - tutor config save tutor plugins enable codejail tutor local do init --limit codejail - tutor mounts add cms:mitx-grading-library/integration_tests/integration_test.py:/openedx/edx-platform/integration_test.py - tutor local launch -I + shell: bash + + - name: Mount Integration Test + run: tutor mounts add cms:mitx-grading-library/integration_tests/integration_test.py:/openedx/edx-platform/integration_test.py + shell: bash + + - name: Launch Tutor + run: tutor local launch -I shell: bash - name: Import MITx Demo Course @@ -48,5 +57,5 @@ jobs: - name: Run integration tests run: | - tutor local run cms python3 integration_test.py + tutor local run cms python integration_test.py "$COURSE_KEY" shell: bash diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py index 6ee7ac0..1f727be 100644 --- a/integration_tests/integration_test.py +++ b/integration_tests/integration_test.py @@ -4,13 +4,15 @@ to be able to use the python_lib.zip that contains the library. """ +import logging import os +import sys + import django -import logging +from opaque_keys.edx.keys import CourseKey from xmodule.capa.safe_exec import safe_exec -from xmodule.util.sandboxing import SandboxService from xmodule.contentstore.django import contentstore -from opaque_keys.edx.keys import CourseKey +from xmodule.util.sandboxing import SandboxService # Set up Django environment os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.test") @@ -19,7 +21,7 @@ log = logging.getLogger(__name__) # Define the code to be executed -all_code = """ +GRADING_CLASSES_CODE = """ from mitxgraders import * # Grading Classes @@ -91,12 +93,11 @@ """ -def execute_code(all_code, course_key_str): +def execute_code(course_key_str): """ Executes the provided code in a sandboxed environment with the specified course context. Args: - all_code (str): The code to be executed. course_key_str (str): The string representation of the course key. Returns: @@ -105,7 +106,8 @@ def execute_code(all_code, course_key_str): course_key = CourseKey.from_string(course_key_str) sandbox_service = SandboxService( course_id=course_key, - contentstore=contentstore) + contentstore=contentstore + ) zip_lib = sandbox_service.get_python_lib_zip() extra_files = [] @@ -116,7 +118,7 @@ def execute_code(all_code, course_key_str): python_path.append("python_lib.zip") safe_exec( - all_code, + code=GRADING_CLASSES_CODE, globals_dict={}, python_path=python_path, extra_files=extra_files, @@ -127,5 +129,9 @@ def execute_code(all_code, course_key_str): if __name__ == "__main__": - course_key_str = "course-v1:MITx+grading-library+course" - execute_code(all_code, course_key_str) + if len(sys.argv) != 2: + print("Usage: python integration_test.py ") + sys.exit(1) + + course_key_str = sys.argv[1] + execute_code(course_key_str) From c4d48da600e77b01bd2710498db19a69ce9e8a39 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 30 Dec 2024 20:34:59 -0500 Subject: [PATCH 06/10] fix: test without codejail --- .github/workflows/integration-test.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 0c6aeb4..13d6a0c 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -34,13 +34,6 @@ jobs: - name: Install Tutor run: pip install "tutor${{ matrix.tutor_version }}" shell: bash - - - name: Install, enable and initialize Tutor Codejail Plugin - run: | - pip install git+https://github.com/edunext/tutor-contrib-codejail - tutor plugins enable codejail - tutor local do init --limit codejail - shell: bash - name: Mount Integration Test run: tutor mounts add cms:mitx-grading-library/integration_tests/integration_test.py:/openedx/edx-platform/integration_test.py From 37d5c39f64572b28ca897c22672377618f86f014 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 30 Dec 2024 20:59:21 -0500 Subject: [PATCH 07/10] Revert "fix: test without codejail" This reverts commit c4d48da600e77b01bd2710498db19a69ce9e8a39. --- .github/workflows/integration-test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 13d6a0c..0c6aeb4 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -34,6 +34,13 @@ jobs: - name: Install Tutor run: pip install "tutor${{ matrix.tutor_version }}" shell: bash + + - name: Install, enable and initialize Tutor Codejail Plugin + run: | + pip install git+https://github.com/edunext/tutor-contrib-codejail + tutor plugins enable codejail + tutor local do init --limit codejail + shell: bash - name: Mount Integration Test run: tutor mounts add cms:mitx-grading-library/integration_tests/integration_test.py:/openedx/edx-platform/integration_test.py From a667906c45dcd3fc657b7a4546fecdeb9126dff4 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 2 Jan 2025 12:15:30 -0500 Subject: [PATCH 08/10] fix: test that we can use the graders --- integration_tests/integration_test.py | 76 ++++----------------------- 1 file changed, 10 insertions(+), 66 deletions(-) diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py index 1f727be..acc7f6d 100644 --- a/integration_tests/integration_test.py +++ b/integration_tests/integration_test.py @@ -22,72 +22,16 @@ # Define the code to be executed GRADING_CLASSES_CODE = """ -from mitxgraders import * - -# Grading Classes - -## Single-input graders - -StringGrader(answers="cat") -FormulaGrader(answers='0') -NumericalGrader() -MatrixGrader( - answers='x*A*B*u + z*C^3*v/(u*C*v)', - variables=['A', 'B', 'C', 'u', 'v', 'z', 'x'], - sample_from={ - 'A': RealMatrices(shape=[2, 3]), - 'B': RealMatrices(shape=[3, 2]), - 'C': RealMatrices(shape=[2, 2]), - 'u': RealVectors(shape=[2]), - 'v': RealVectors(shape=[2]), - 'z': ComplexRectangle() - }, - identity_dim=2 -) -SingleListGrader( - answers=['cat', 'dog', 'unicorn'], - subgrader=StringGrader() -) - -## Multi-input graders - -ListGrader( - answers=['cat', 'dog', 'unicorn'], - subgraders=StringGrader() -) - -## Specialized graders - -IntegralGrader( - answers={ - 'lower': 'a', - 'upper': 'b', - 'integrand': 'x^2', - 'integration_variable': 'x' - }, - input_positions={ - 'integrand': 1, - 'lower': 2, - 'upper': 3 - }, - variables=['a', 'b'] -) - -IntervalGrader(answers=['(','1','2',']']) - -SumGrader( - answers={ - 'lower': 'a', - 'upper': 'b', - 'summand': 'x^2', - 'summation_variable': 'x' - }, - input_positions={ - 'summand': 1, - 'lower': 2, - 'upper': 3 - }, - variables=['a', 'b'] +from mitxgraders import ( + StringGrader, + FormulaGrader, + NumericalGrader, + MatrixGrader, + SingleListGrader, + ListGrader, + IntegralGrader, + IntervalGrader, + SumGrader, ) """ From df52ccee6350ccaba55d74f80b3309d436b1f95e Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 2 Jan 2025 12:38:53 -0500 Subject: [PATCH 09/10] fix: add matrixgrader in the test --- integration_tests/integration_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/integration_tests/integration_test.py b/integration_tests/integration_test.py index acc7f6d..9456718 100644 --- a/integration_tests/integration_test.py +++ b/integration_tests/integration_test.py @@ -22,6 +22,7 @@ # Define the code to be executed GRADING_CLASSES_CODE = """ +# To test, we can use the custom graders from mitxgraders import ( StringGrader, FormulaGrader, @@ -32,6 +33,24 @@ IntegralGrader, IntervalGrader, SumGrader, + RealMatrices, + RealVectors, + ComplexRectangle, +) + +# Test the MatrixGrader, the class that uses the most external dependencies +MatrixGrader( + answers='x*A*B*u + z*C^3*v/(u*C*v)', + variables=['A', 'B', 'C', 'u', 'v', 'z', 'x'], + sample_from={ + 'A': RealMatrices(shape=[2, 3]), + 'B': RealMatrices(shape=[3, 2]), + 'C': RealMatrices(shape=[2, 2]), + 'u': RealVectors(shape=[2]), + 'v': RealVectors(shape=[2]), + 'z': ComplexRectangle() + }, + identity_dim=2 ) """ From 6272d475ad5c638687d33b0abdcca66e8d5cb40b Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 2 Jan 2025 12:57:13 -0500 Subject: [PATCH 10/10] fix: run the action monthly --- .github/workflows/integration-test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 0c6aeb4..46f4aaf 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -1,6 +1,9 @@ name: Integration Test -on: [pull_request] +on: + schedule: + # Montlhy at 00:00 + - cron: '0 0 1 * *' jobs: edx-platform-integration-test: