Skip to content

Commit

Permalink
Merge branch 'master' into upgrade-pymongo-
Browse files Browse the repository at this point in the history
  • Loading branch information
awais786 authored May 21, 2024
2 parents c9d66dd + 362899e commit f72abaf
Show file tree
Hide file tree
Showing 220 changed files with 5,012 additions and 2,698 deletions.
8 changes: 4 additions & 4 deletions .github/actions/verify-tests-count/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ runs:
- name: collect tests from all modules
shell: bash
run: |
echo "root_cms_unit_tests_count=$(pytest --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "root_lms_unit_tests_count=$(pytest --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
- name: get GHA unit test paths
shell: bash
Expand All @@ -19,8 +19,8 @@ runs:
- name: collect tests from GHA unit test shards
shell: bash
run: |
echo "cms_unit_tests_count=$(pytest --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "lms_unit_tests_count=$(pytest --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
- name: add unit tests count
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/check-consistent-dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
echo $'Paths touched in PR:\n'"$paths"
# The ^"? is because git may quote weird file paths
matched="$(echo "$paths" | grep -P '^"?requirements/' || true)"
matched="$(echo "$paths" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))' || true)"
echo $'Relevant paths:\n'"$matched"
if [[ -n "$matched" ]]; then
echo "RELEVANT=true" >> "$GITHUB_ENV"
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/ci-static-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: ['3.8']
python-version:
- '3.11'
os: ['ubuntu-20.04']

steps:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/compile-python-requirements.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
- name: Set up Python environment
uses: actions/setup-python@v4
with:
python-version: "3.8"
python-version: "3.11"

- name: Run make compile-requirements
env:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/js-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ jobs:
matrix:
os: [ ubuntu-20.04 ]
node-version: [ 18 ]
python-version: [ 3.8 ]
python-version:
- '3.11'

steps:

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/lint-imports.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
python-version: '3.11'

- name: Install system requirements
run: sudo apt update && sudo apt install -y libxmlsec1-dev
Expand Down
10 changes: 6 additions & 4 deletions .github/workflows/migrations-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ jobs:
strategy:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
python-version:
- '3.11'
# 'pinned' is used to install the latest patch version of Django
# within the global constraint i.e. Django==4.2.8 in current case
# because we have global constraint of Django<4.2
# because we have global constraint of Django<4.2
django-version: ["pinned"]
mongo-version: ["4", "7"]
mongo-version:
- "7"
mysql-version: ["8"]
services:
mongo:
Expand Down Expand Up @@ -115,7 +117,7 @@ jobs:
./manage.py lms migrate
echo "Running the CMS migrations."
./manage.py cms migrate
# This job aggregates test results. It's the required check for branch protection.
# https://github.com/marketplace/actions/alls-green#why
# https://github.com/orgs/community/discussions/33579
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pylint-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.8
python-version: 3.11

- name: Get pip cache dir
id: pip-cache-dir
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/quality-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ jobs:
strategy:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
node-version: [ 16 ]
python-version:
- '3.11'
node-version: [ 18 ]

steps:

Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/semgrep.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ jobs:
strategy:
matrix:
os: [ "ubuntu-20.04" ]
python-version: [ "3.8" ]
python-version:
- '3.11'

steps:
- uses: actions/checkout@v3
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/static-assets-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ jobs:
strategy:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
python-version:
- '3.11'
node-version: [ 18 ]
npm-version: [ 10.5.x ]
mongo-version: ["4.4", "7.0"]
mongo-version:
- "7.0"

services:
mongo:
Expand Down
25 changes: 20 additions & 5 deletions .github/workflows/unit-tests-gh-hosted.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ on:

jobs:
run-test:
name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true))
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python-version: [ '3.8' ]
python-version:
- "3.11"
django-version:
- "pinned"
# When updating the shards, remember to make the same changes in
Expand All @@ -37,9 +38,23 @@ jobs:
- "xmodule-with-lms"
- "xmodule-with-cms"
mongo-version:
- "4.4"
- "7.0"
name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }}${{ matrix.shard_name }}

# We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
# This code is left here as an example for future refernce in case we need to reduce the number of shards we're
# testing but still have good confidence with older versions of mongo. We use Mongo 4.4 in the example.
#
# exclude:
# - mongo-version: "4.4"
# include:
# - shard_name: "xmodule-with-cms"
# python-version: "3.11"
# django-version: "pinned"
# mongo-version: "4.4"
# - shard_name: "xmodule-with-lms"
# python-version: "3.11"
# django-version: "pinned"
# mongo-version: "4.4"
steps:
- uses: actions/checkout@v2

Expand Down Expand Up @@ -87,7 +102,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [ '3.8' ]
python-version: [ '3.11' ]
django-version:
- "pinned"
steps:
Expand Down
19 changes: 12 additions & 7 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ on:

jobs:
run-tests:
name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }},${{ matrix.shard_name }}
name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
runs-on: [ edx-platform-runner ]
strategy:
matrix:
python-version:
- "3.8"
- "3.11"
django-version:
- "pinned"
# When updating the shards, remember to make the same changes in
Expand All @@ -37,7 +37,6 @@ jobs:
- "xmodule-with-lms"
- "xmodule-with-cms"
mongo-version:
- "4.4"
- "7.0"
# We expect Django 4.0 to fail, so don't stop when it fails.
continue-on-error: ${{ matrix.django-version == 'pinned' }}
Expand All @@ -56,15 +55,20 @@ jobs:
sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
fi
- name: checkout repo
uses: actions/checkout@v3

- name: start mongod server for tests
run: |
sudo mkdir -p /data/db
sudo chmod -R a+rw /data/db
mongod &
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: checkout repo
uses: actions/checkout@v3

- name: install requirements
run: |
sudo make test-requirements
Expand Down Expand Up @@ -140,7 +144,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ 3.8 ]
python-version:
- 3.11
steps:
- name: Checkout repo
uses: actions/checkout@v3
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/units-test-scripts-structures-pruning.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ jobs:

strategy:
matrix:
python-version: [ '3.8', '3.12' ]
python-version:
- '3.12'

steps:
- name: Checkout code
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/units-test-scripts-user-retirement.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ jobs:

strategy:
matrix:
python-version: [ '3.8' ]
python-version:
- '3.12'

steps:
- name: Checkout code
Expand Down
98 changes: 98 additions & 0 deletions .github/workflows/update-geolite-database.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
name: Update GeoLite Database

on:
schedule:
- cron: "0 3 1 * *"
workflow_dispatch:
inputs:
branch:
description: 'Target branch against which to create PR'
required: false
default: 'master'

env:
MAXMIND_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz'
MAXMIND_SHA256_URL: 'https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${{ secrets.MAXMIND_LICENSE_KEY }}&suffix=tar.gz.sha256'
TAR_FILE_NAME: 'GeoLite2-Country.tar.gz'
TAR_SHA256_FILE_NAME: 'GeoLite2-Country.tar.gz.sha256'
TAR_UNZIPPED_ROOT_PATTERN: 'GeoLite2-Country_*'
DB_FILE: 'GeoLite2-Country.mmdb'
DB_DESTINATION_PATH: 'common/static/data/geoip'

jobs:
download-and-replace:
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@v4

- name: Download GeoLite tar file
run: |
wget -O '${{ env.TAR_FILE_NAME }}' '${{ env.MAXMIND_URL }}'\
- name: Download GeoLite sha256 file
run: |
wget -O '${{ env.TAR_SHA256_FILE_NAME }}' '${{ env.MAXMIND_SHA256_URL }}'\
- name: Check SHA256 hash
run: |
sha256sum '${{ env.TAR_FILE_NAME }}' | grep $(cat '${{ env.TAR_SHA256_FILE_NAME }}' | cut -d' ' -f1)
- name: Extract tar file
run: |
tar xvf '${{ env.TAR_FILE_NAME }}'
- name: Copy DB file to destination path
run: |
find . -type d -name '${{ env.TAR_UNZIPPED_ROOT_PATTERN }}' -exec cp {}/'${{ env.DB_FILE }}' '${{ env.DB_DESTINATION_PATH }}'/ \;
- name: Delete un-required content
run: |
rm '${{ env.TAR_FILE_NAME }}'
rm '${{ env.TAR_SHA256_FILE_NAME }}'
find . -type d -name '${{ env.TAR_UNZIPPED_ROOT_PATTERN }}' -exec rm -r {} \; || true
- name: PR preflight
run: |
if git diff --exit-code; then
echo 'Summary: No updates/changes detected. Terminating the run and no pull request is going to be created.' | tee -a '$GITHUB_STEP_SUMMARY'
exit 1
else
echo 'Updates/changes detected, going to create PR.'
fi
- name: Setup git
run: |
git config --global user.email '${{ github.actor }}@users.noreply.github.com'
git config --global user.name '${{ github.actor }}'
- name: Create a branch, commit the code and make a PR
id: create-pr
run: |
BRANCH="${{ github.actor }}/geoip2-bot-update-country-database-$(echo "${{ github.sha }}" | cut -c 1-7)"
git checkout -b $BRANCH
git add .
git status
git commit -m "chore: geoip2: update maxmind geolite country database"
git push --set-upstream origin $BRANCH
PR_URL=$(gh pr create \
--title "Update GeoLite Database" \
--body "PR generated by workflow `${{ github.workflow }}` on behalf of @${{ github.actor }}." \
--head $BRANCH \
--base 'master' \
--reviewer 'feanil' \
| grep -o 'https://github.com/.*/pull/[0-9]*')
echo "PR Created: ${PR_URL}"
echo "pull-request-url=$PR_URL" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}

- name: Job summary
run: |
PR_URL=${{ steps.create-pr.outputs.pull-request-url }}
if [[ -z '$PR_URL' ]]; then
echo 'Error: PR creation unsuccessful; refer to the log for further details.' | tee -a "${GITHUB_STEP_SUMMARY}"
exit 1
else
echo "PR: ${PR_URL}" | tee -a "${GITHUB_STEP_SUMMARY}"
fi
8 changes: 5 additions & 3 deletions .github/workflows/upgrade-one-python-dependency.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
- name: Set up Python environment
uses: actions/setup-python@v4
with:
python-version: "3.8"
python-version: "3.11"

- name: Update any pinned dependencies
env:
Expand Down Expand Up @@ -82,11 +82,13 @@ jobs:
- name: Make a PR
id: make-pr
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
branch: "${{ github.triggering_actor }}/upgrade-${{ inputs.package }}"
branch-suffix: short-commit-hash
add-paths: requirements
add-paths: |
requirements
scripts/**/requirements*
commit-message: |
feat: Upgrade Python dependency ${{ inputs.package }}
Expand Down
Loading

0 comments on commit f72abaf

Please sign in to comment.