Merge main into release

.env.example

@@ -1,3 +1,9 @@
+POSTGRES_PORT=5420
+APISIX_PORT=9080
+KEYCLOAK_SSL_PORT=7443
+KEYCLOAK_PORT=7080
+NGINX_PORT=8073
+
 SITE_NAME="MIT Open Learning Ecommerce"
 SECRET_KEY=
 STATUS_TOKEN=
@@ -46,5 +52,6 @@ KEYCLOAK_DISCOVERY_URL=
 KEYCLOAK_CLIENT_ID=
 KEYCLOAK_CLIENT_SECRET=
 
-APISIX_PORT=9080
 APISIX_SESSION_SECRET_KEY=must_be_at_least_16_chars
+
+MITOL_LEARN_API_URL=https://api.rc.learn.mit.edu/api/v1/

.github/workflows/ci.yml

@@ -78,3 +78,50 @@ jobs:
         uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
         with:
           file: ./coverage.xml
+
+  openapi-generated-client-check-v0:
+    # This job checks that the output of openapi-generator-typescript-axios that
+    # is checked into version control is up-to-date.
+    env:
+      OPENAPI_SCHEMA: ./openapi/specs/v0.yaml
+      GENERATOR_IGNORE_FILE: ./frontends/api/.openapi-generator-ignore
+      GENERATOR_OUTPUT_DIR_CI: ./frontends/api/tmp/generated/v0
+      GENERATOR_OUTPUT_DIR_VC: ./frontends/api/src/generated/v0
+      COREPACK_ENABLE_DOWNLOAD_PROMPT: 0
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
+        with:
+          node-version: "^20"
+          cache: "npm"
+          cache-dependency-path: "package-lock.json"
+
+      - name: Install dependencies
+        run: npm i
+
+      - name: Generate Fresh API Client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+          generator: typescript-axios
+          openapi-file: $OPENAPI_SCHEMA
+          generator-tag: v7.2.0
+          command-args: |
+            --output $GENERATOR_OUTPUT_DIR_CI \
+            --ignore-file-override $GENERATOR_IGNORE_FILE \
+            -c scripts/openapi-configs/typescript-axios-v0.yaml
+
+      - name: Format freshly generated client
+        run: npx prettier $GENERATOR_OUTPUT_DIR_CI/**/*.ts --no-semi --write
+
+      # Kept getting tripped up on npmignore and the git push script, neither
+      # of which are really important, so just check the generated TypeScript
+      # files instead.
+      - name: Check VC client is up-to-date
+        run: |
+          mkdir $GENERATOR_OUTPUT_DIR_VC/ts
+          mkdir $GENERATOR_OUTPUT_DIR_CI/ts
+          cp $GENERATOR_OUTPUT_DIR_VC/*.ts $GENERATOR_OUTPUT_DIR_VC/ts
+          cp $GENERATOR_OUTPUT_DIR_CI/*.ts $GENERATOR_OUTPUT_DIR_CI/ts
+          diff $GENERATOR_OUTPUT_DIR_CI/ts $GENERATOR_OUTPUT_DIR_VC/ts \
+          || { echo "OpenAPI spec is out of date. Please regenerate via ./scripts/generate_openapi.sh"; exit 1; }

.pre-commit-config.yaml

@@ -1,8 +1,6 @@
 ---
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-# A bunch of frontend related stuff is commented out here - this is because we don't
-# have a frontend yet.
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
@@ -15,8 +13,29 @@ repos:
       - id: check-merge-conflict
       - id: check-toml
       - id: debug-statements
+  - repo: local
+    hooks:
+      - id: prettier
+        name: prettier
+        entry: npx prettier --write
+        language: node
+        types_or:
+          [
+            javascript,
+            jsx,
+            ts,
+            tsx,
+            json,
+            scss,
+            sass,
+            css,
+            yaml,
+            markdown,
+            html,
+            sh,
+          ]
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.8.0-1
+    rev: v3.10.0-1
     hooks:
       - id: shfmt
   - repo: https://github.com/adrienverge/yamllint.git
@@ -58,3 +77,5 @@ repos:
     hooks:
       - id: shellcheck
         args: ["--severity=warning"]
+ci:
+  skip: [prettier]

.prettierrc.json

@@ -1,5 +1,5 @@
 {
   "$schema": "http://json.schemastore.org/prettierrc",
   "semi": false,
-  "plugins": ["prettier-plugin-django-alpine"]
+  "plugins": ["prettier-plugin-sh"]
 }

.secrets.baseline

@@ -175,15 +175,6 @@
         "line_number": 1
       }
     ],
-    "docker-compose-apisix.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": "docker-compose-apisix.yml",
-        "hashed_secret": "965f383cb0ba71f9e9d33ae7eec9380590a4e9e3",
-        "is_verified": false,
-        "line_number": 43
-      }
-    ],
     "docker-compose.yml": [
       {
         "type": "Basic Auth Credentials",
@@ -194,5 +185,5 @@
       }
     ]
   },
-  "generated_at": "2024-11-21T20:53:59Z"
+  "generated_at": "2024-12-05T14:34:37Z"
 }

README.md

@@ -16,6 +16,8 @@ This application provides a central system to handle ecommerce activities across
     - [Interstitial Debug Mode](#interstitial-debug-mode)
     - [Webhook Retry](#webhook-retry)
     - [Running the app in a notebook](#running-the-app-in-a-notebook)
+    - [Change ports](#change-ports)
+    - [Import MaxMind GeoIP data](#import-maxmind-geoip-data)
 
 ## Initial Setup
 
@@ -184,3 +186,31 @@ This repo includes a config for running a [Jupyter notebook](https://jupyter.org
 - Execute the first block to confirm it's working properly (click inside the block and press Shift+Enter)
 
 From there, you should be able to run code snippets with a live Django app just like you would in a Django shell.
+
+### Change ports
+
+If you need, you can change the exposed ports for all services:
+
+```
+POSTGRES_PORT=5420
+APISIX_PORT=9080
+KEYCLOAK_SSL_PORT=7443
+KEYCLOAK_PORT=7080
+NGINX_PORT=8073
+```
+
+If you change these, you may need to update settings elsewhere. (Note that the APISIX config references `nginx:8073` but since it's _within_ the Docker network for the app, you don't need to update its port if you change it in your `.env` file.)
+
+### Import MaxMind GeoIP data
+
+The blocked country and tax assessment checks need the MaxMind GeoLite2 dataset to be imported into the app.
+
+You'll need to retrieve a copy of the data. You can get this for free from MaxMind: https://dev.maxmind.com/ Use the blue "Sign up for a GeoLite2 account" at the bottom to sign up for an account, and then you can download the data. There are several versions of the data to download - generally the "Country: CSV Format" is the best option. (You _have_ to use a CSV option, however.)
+
+Once you've downloaded it, place the CSV files in the root directory and then you can run this one-liner:
+
+```
+docker compose exec web ./manage.py import_maxmind_data GeoLite2-Country-Locations-en.csv geolite2-country-locations ; docker compose exec web ./manage.py import_maxmind_data GeoLite2-Country-Blocks-IPv4.csv geolite2-country-ipv4 ; docker compose exec web ./manage.py import_maxmind_data GeoLite2-Country-Blocks-IPv6.csv geolite2-country-ipv6
+```
+
+You can also (and probably should) add mappings for private IPs too. Private IPs aren't represented by default in the GeoIP databases. Run `docker compose exec web ./manage.py create_private_maxmind_data <ISO code>` to do this. The ISO code can be anything that's a valid ISO 3166 code (so, US works, but you can set it to something else if you'd prefer).

cart/templates/cart.html

@@ -96,7 +96,9 @@
                 <select name="product" id="product">
                     <option></option>
                     {% for product in products %}
+                    {% if product.system.id == basket.integrated_system.id %}
                     <option value="{{ product.system.slug }}/{{ product.sku }}">{{ product.system.name }} - {{ product.name }} ${{ product.price }}</option>
+                    {% endif %}
                     {% endfor %}
                 </select>
             </div>

config/apisix/apisix.yaml

@@ -27,6 +27,7 @@ routes:
     - "/auth/*"
     - "/static/*"
     - "/favicon.ico"
+    - "/checkout/*"
   - id: 2
     name: "ue-default"
     desc: "Wildcard route for the rest of the system - authentication required"

conftest.py

@@ -1,13 +1,37 @@
-"""Project conftest"""
-
 # pylint: disable=wildcard-import, unused-wildcard-import
+from unittest.mock import Mock, patch
+
 import pytest
 
 from fixtures.common import *  # noqa: F403
 from fixtures.users import *  # noqa: F403
 from unified_ecommerce.exceptions import DoNotUseRequestException
 
 
+@pytest.fixture(autouse=True)
+def mock_requests_get():
+    # Mock the response of requests.get
+    with patch("system_meta.tasks.requests.get") as mock_get:
+        # Create a mock response object
+        mock_response = Mock()
+        mock_response.raise_for_status = Mock()  # Mock the raise_for_status method
+        mock_response.json.return_value = {
+            "results": [
+                {
+                    "image": {
+                        "url": "http://example.com/image.jpg",
+                        "alt": "Image alt text",
+                        "description": "Image description",
+                    }
+                }
+            ]
+        }
+        mock_get.return_value = (
+            mock_response  # Set the mock response to be returned by requests.get
+        )
+        yield mock_get  # This will be the mocked requests.get
+
+
 @pytest.fixture(autouse=True)
 def prevent_requests(mocker, request):  # noqa: PT004
     """Patch requests to error on request by default"""

docker-compose.yml

@@ -12,17 +12,21 @@ x-environment: &py-environment
   DOCKER_HOST: ${DOCKER_HOST:-missing}
   POSTGRES_PORT: ${POSTGRES_PORT:-5432}
   POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+  NGINX_PORT: ${NGINX_PORT:-8073}
+  APISIX_PORT: ${APISIX_PORT:-9080}
+  KEYCLOAK_PORT: ${KEYCLOAK_PORT:-7080}
+  KEYCLOAK_SSL_PORT: ${KEYCLOAK_SSL_PORT:-7443}
 
 services:
   db:
-    image: postgres:16
+    image: postgres:17
     ports:
       - "$POSTGRES_PORT:5432"
     environment:
       <<: *py-environment
 
   redis:
-    image: redis:5.0.14
+    image: redis:6.2
     ports:
       - "6379"
 
@@ -49,7 +53,7 @@ services:
     stdin_open: true
     tty: true
     ports:
-      - "8071:8071"
+      - "8071"
     links:
       - db
       - redis
@@ -86,8 +90,7 @@ services:
     - APISIX_SESSION_SECRET_KEY=${APISIX_SESSION_SECRET_KEY:-something_at_least_16_characters}
     - UE_LOGOUT_URL=${UE_LOGOUT_URL:-http://ue.odl.local:9080/auth/logout/}
     ports:
-      - 9080:9080
-      - 9180:9180
+      - ${APISIX_PORT}:9080
     volumes:
       - ./config/apisix/config.yaml:/usr/local/apisix/conf/config.yaml
       - ./config/apisix/apisix.yaml:/usr/local/apisix/conf/apisix.yaml
@@ -100,7 +103,7 @@ services:
     depends_on:
       - db
     ports:
-      - 7080:7080
+      - ${KEYCLOAK_PORT}:7080
       - 7443:7443
     environment:
       - KEYCLOAK_ADMIN=${KEYCLOAK_SVC_ADMIN:-admin}
@@ -111,7 +114,7 @@ services:
         - ${KEYCLOAK_SVC_HOSTNAME:-kc.odl.local}
     links:
       - db:uedb
-    command: start --verbose --features scripts --import-realm --hostname=${KEYCLOAK_SVC_HOSTNAME:-kc.odl.local} --hostname-strict=false --hostname-debug=true --https-port=7443 --https-certificate-file=/etc/x509/https/tls.crt --https-certificate-key-file=/etc/x509/https/tls.key --http-enabled=true --http-port=7080 --config-keystore=/etc/keycloak-store --config-keystore-password=${KEYCLOAK_SVC_KEYSTORE_PASSWORD} --db=postgres --db-url-database=keycloak --db-url-host=uedb --db-schema=public --db-password=${POSTGRES_PASSWORD:-postgres} --db-username=postgres
+    command: start --verbose --features scripts --import-realm --hostname=${KEYCLOAK_SVC_HOSTNAME:-kc.odl.local} --hostname-strict=false --hostname-debug=true --https-port=${KEYCLOAK_SSL_PORT} --https-certificate-file=/etc/x509/https/tls.crt --https-certificate-key-file=/etc/x509/https/tls.key --http-enabled=true --http-port=${KEYCLOAK_PORT} --config-keystore=/etc/keycloak-store --config-keystore-password=${KEYCLOAK_SVC_KEYSTORE_PASSWORD} --db=postgres --db-url-database=keycloak --db-url-host=uedb --db-schema=public --db-password=${POSTGRES_PASSWORD:-postgres} --db-username=postgres
     volumes:
       - keycloak-store:/etc/keycloak-store
       - ./config/keycloak/tls:/etc/x509/https

frontends/api/src/generated/v0/.openapi-generator-ignore

@@ -21,8 +21,3 @@
 #docs/*.md
 # Then explicitly reverse the ignore rule for a single file:
 #!docs/README.md
-
-**/.npmignore
-**/.gitignore
-**/git_push.sh
-**/.openapi-generator-ignore