Allow manual CI for Docker building #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and push Docker Images from modified configuration.yaml | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| jobs: | |
| extract_changed_files: | |
| name: Find changed configurations | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| outputs: | |
| matrix: ${{ steps.create-json.outputs.matrix }} | |
| steps: | |
| - name: Clone DL4MicEverywhere repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. | |
| - name: Get changed configuration.yaml on the commint | |
| id: changed-files | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files: '**/configuration.yaml' | |
| - if: ${{ steps.changed-files.outputs.all_changed_files != '' }} | |
| name: List and extract all changed files | |
| id: create-json | |
| run: | | |
| echo "Start" | |
| files_json="{\"include\":[{\"notebook\":\"" | |
| echo "$files_json" | |
| first=true | |
| for file in ${{ steps.changed-files.outputs.all_changed_files }}; do | |
| echo "$file was changed" | |
| if [ "$first" = true ]; then | |
| files_json+=$file | |
| first=false | |
| else | |
| files_json+="\"},{\"notebook\":\""$file | |
| fi | |
| echo "$files_json" | |
| done | |
| files_json+="\"}]}" | |
| echo "$files_json" | |
| echo "matrix=$files_json" >> $GITHUB_OUTPUT | |
| echo "END" | |
| check_keys: | |
| name: Check attributes/keys on modified configuration.yaml | |
| if: ${{ needs.extract_changed_files.outputs.matrix != '' }} | |
| runs-on: [ubuntu-latest] | |
| strategy: | |
| # put fail-fast as false if you want to see all results even if one fails, | |
| # fail-fast is true by default | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.extract_changed_files.outputs.matrix) }} | |
| needs: [extract_changed_files] | |
| outputs: | |
| ubuntu_version: ${{ steps.ubuntu_version.outputs.result }} | |
| cuda_version: ${{ steps.cuda_version.outputs.result }} | |
| cudnn_version: ${{ steps.cudnn_version.outputs.result }} | |
| python_version: ${{ steps.python_version.outputs.result }} | |
| notebook_url: ${{ steps.notebook_url.outputs.result }} | |
| requirements_url: ${{ steps.requirements_url.outputs.result }} | |
| notebook_name: ${{ steps.notebook_name.outputs.value }} | |
| docker_tag: ${{ steps.docker_tag.outputs.value }} | |
| sections_to_remove: ${{ steps.sections_to_remove.outputs.result }} | |
| description: ${{ steps.description.outputs.result }} | |
| steps: | |
| - name: Clone DL4MicEverywhere repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| cache: 'pip' # caching pip dependencies | |
| - name: Install requirements | |
| run: pip install -r .tools/requirements.txt | |
| - name: Update the docker_tag on the configuration | |
| run: python3 .tools/python_tools/update_configuration.py '${{ matrix.notebook }}' | |
| - name: Check notebook_url attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: notebook_url | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.notebook_url' '${{ matrix.notebook }}' | |
| - name: Check requirements_url attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: requirements_url | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.requirements_url' '${{ matrix.notebook }}' | |
| - name: Check cuda_version attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: cuda_version | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.cuda_version' '${{ matrix.notebook }}' | |
| - name: Check cudnn_version attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: cudnn_version | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.cudnn_version' '${{ matrix.notebook }}' | |
| - name: Check ubuntu_version attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: ubuntu_version | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.ubuntu_version' '${{ matrix.notebook }}' | |
| - name: Check python_version attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: python_version | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.python_version' '${{ matrix.notebook }}' | |
| - name: Check sections_to_remove attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: sections_to_remove | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.sections_to_remove' '${{ matrix.notebook }}' | |
| - name: Check docker_tag attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: docker_tag | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.docker_hub_image' '${{ matrix.notebook }}' | |
| - name: Check description attribute on configuration.yaml | |
| uses: mikefarah/yq@master | |
| id: description | |
| with: | |
| cmd: yq '.config.dl4miceverywhere.description' '${{ matrix.notebook }}' | |
| - name: Get notebook_name | |
| id: notebook_name | |
| run: | | |
| notebook_name="$(basename ${{ steps.notebook_url.outputs.result }})" | |
| echo "value=$notebook_name" >> $GITHUB_OUTPUT | |
| - run: | | |
| if [[ 'null' = '${{ steps.notebook_url.outputs.result }}' ]] ; then | |
| echo 'The key notebook_url is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.requirements_url.outputs.result }}' ]] ; then | |
| echo 'The key requirements_url is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.cuda_version.outputs.result }}' ]] ; then | |
| echo 'The key cuda_version is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.cudnn_version.outputs.result }}' ]] ; then | |
| echo 'The key cudnn_version is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.ubuntu_version.outputs.result }}' ]] ; then | |
| echo 'The key ubuntu_version is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.python_version.outputs.result }}' ]] ; then | |
| echo 'The key python_version is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.sections_to_remove.outputs.result }}' ]] ; then | |
| echo 'The key sections_to_remove is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.docker_tag.outputs.result }}' ]] ; then | |
| echo 'The key docker_hub_image is not in the configuration.yaml file.' | |
| exit 1 | |
| elif [[ 'null' = '${{ steps.description.outputs.result }}' ]] ; then | |
| echo 'The key description is not in the configuration.yaml file.' | |
| exit 1 | |
| else | |
| echo 'The configuration.yaml file is defined with all its keys.' | |
| fi | |
| check_urls: | |
| name: Check notebook's and requirements' URLs | |
| if: ${{ needs.extract_changed_files.outputs.matrix != '' }} | |
| runs-on: [ubuntu-latest] | |
| strategy: | |
| # super important if you want to see all results, even if one fails | |
| # fail-fast is true by default | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.extract_changed_files.outputs.matrix) }} | |
| needs: [check_keys, extract_changed_files] | |
| steps: | |
| - name: Clone DL4MicEverywhere repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check that notebook's URL exists | |
| uses: filiph/[email protected] | |
| with: | |
| arguments: ${{ needs.check_keys.outputs.notebook_url }} | |
| - name: Check that requirements' URL exists | |
| uses: filiph/[email protected] | |
| with: | |
| arguments: ${{ needs.check_keys.outputs.requirements_url }} | |
| configuration_matrix: | |
| name: Call the building and pushing | |
| if: ${{ needs.extract_changed_files.outputs.matrix != '' }} | |
| needs: extract_changed_files | |
| strategy: | |
| # super important if you want to see all results, even if one fails | |
| # fail-fast is true by default | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.extract_changed_files.outputs.matrix) }} | |
| uses: ./.github/workflows/build_docker_images_aux.yml # calls the one above ^ | |
| with: | |
| config_file: ${{ matrix.notebook }} | |
| secrets: inherit | |