Solve database lock during long computations by breaking up the make callback into three parts #1241
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Development | |
| on: | |
| push: | |
| branches: | |
| - "**" # every branch | |
| - "!gh-pages" # exclude gh-pages branch | |
| - "!stage*" # exclude branches beginning with stage | |
| tags: | |
| - '\d+\.\d+\.\d+' # only semver tags | |
| pull_request: | |
| branches: | |
| - "**" # every branch | |
| - "!gh-pages" # exclude gh-pages branch | |
| - "!stage*" # exclude branches beginning with stage | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - py_ver: "3.9" | |
| distro: debian | |
| image: djbase | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DISTRO: ${{matrix.distro}} | |
| IMAGE: ${{matrix.image}} | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Validate version and release notes | |
| run: | | |
| DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) | |
| RELEASE_BODY=$(python -c \ | |
| 'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \ | |
| ) | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV | |
| echo "$RELEASE_BODY" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Build pip artifacts | |
| run: | | |
| export HOST_UID=$(id -u) | |
| docker compose -f docker-compose-build.yaml up --exit-code-from app --build | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| - if: matrix.py_ver == '3.9' && matrix.distro == 'debian' | |
| name: Add pip artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: pip-datajoint-${{env.DJ_VERSION}} | |
| path: dist | |
| retention-days: 1 | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| py_ver: ["3.9"] | |
| mysql_ver: ["8.0", "5.7"] | |
| include: | |
| - py_ver: "3.11" | |
| mysql_ver: "8.0" | |
| - py_ver: "3.10" | |
| mysql_ver: "8.0" | |
| - py_ver: "3.8" | |
| mysql_ver: "5.7" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install flake8 black | |
| - name: Run syntax tests | |
| run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Run primary tests | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DJ_PASS: password | |
| MYSQL_VER: ${{matrix.mysql_ver}} | |
| DISTRO: alpine | |
| MINIO_VER: RELEASE.2021-09-03T03-56-13Z | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| run: | | |
| export HOST_UID=$(id -u) | |
| docker compose -f LNX-docker-compose.yml up --build --exit-code-from app | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| py_ver: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install flake8 black==24.2.0 | |
| - name: Run syntax tests | |
| run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Run style tests | |
| run: | | |
| flake8 --ignore=E203,E722,W503 datajoint \ | |
| --count --max-complexity=62 --max-line-length=127 --statistics | |
| black --required-version '24.2.0' --check -v datajoint tests tests_old | |
| codespell: | |
| name: Check for spelling errors | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Codespell | |
| uses: codespell-project/actions-codespell@v2 | |
| publish-docs: | |
| if: | | |
| github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags') | |
| needs: test | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Deploy docs | |
| run: | | |
| export MODE=BUILD | |
| export PACKAGE=datajoint | |
| export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git | |
| export HOST_UID=$(id -u) | |
| docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build | |
| git push origin gh-pages | |
| publish-release: | |
| if: | | |
| github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags') | |
| needs: test | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - py_ver: "3.9" | |
| distro: debian | |
| image: djbase | |
| env: | |
| PY_VER: ${{matrix.py_ver}} | |
| DISTRO: ${{matrix.distro}} | |
| IMAGE: ${{matrix.image}} | |
| TWINE_USERNAME: ${{secrets.twine_username}} | |
| TWINE_PASSWORD: ${{secrets.twine_password}} | |
| DOCKER_CLIENT_TIMEOUT: "120" | |
| COMPOSE_HTTP_TIMEOUT: "120" | |
| outputs: | |
| release_upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{matrix.py_ver}} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{matrix.py_ver}} | |
| - name: Determine package version | |
| run: | | |
| DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) | |
| RELEASE_BODY=$(python -c \ | |
| 'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \ | |
| ) | |
| echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV | |
| echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV | |
| echo "$RELEASE_BODY" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Create GH release | |
| id: create_gh_release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| tag_name: ${{env.DJ_VERSION}} | |
| release_name: Release ${{env.DJ_VERSION}} | |
| body: ${{env.RELEASE_BODY}} | |
| prerelease: false | |
| draft: false | |
| - name: Fetch pip artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pip-datajoint-${{env.DJ_VERSION}} | |
| path: dist | |
| - name: Determine pip artifact paths | |
| run: | | |
| echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV | |
| echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV | |
| - name: Upload pip wheel asset to release | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| asset_path: ${{env.DJ_WHEEL_PATH}} | |
| asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl | |
| asset_content_type: application/zip | |
| - name: Upload pip sdist asset to release | |
| uses: actions/upload-release-asset@v1 | |
| env: | |
| GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} | |
| with: | |
| upload_url: ${{steps.create_gh_release.outputs.upload_url}} | |
| asset_path: ${{env.DJ_SDIST_PATH}} | |
| asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz | |
| asset_content_type: application/gzip | |
| - name: Publish pip release | |
| run: | | |
| export HOST_UID=$(id -u) | |
| docker compose -f docker-compose-build.yaml run \ | |
| -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \ | |
| sh -c "pip install twine && python -m twine upload dist/*" | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{secrets.docker_username}} | |
| password: ${{secrets.docker_password}} | |
| - name: Publish image | |
| run: | | |
| IMAGE=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Repository}}") | |
| TAG=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Tag}}") | |
| docker push "${IMAGE}:${TAG}" | |
| docker tag "${IMAGE}:${TAG}" "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" | |
| docker push "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}" | |
| [ "$PY_VER" == "3.9" ] && [ "$DISTRO" == "debian" ] \ | |
| && docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" \ | |
| && docker push "${IMAGE}:latest" \ | |
| || echo "skipping 'latest' tag..." |