Update python-linear-model-overfitting.ipynb #8
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Changed Jupyter Notebooks | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| test-notebooks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Install git | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y git | |
| shell: bash | |
| - name: Install libgraphviz-dev | |
| run: sudo apt-get install -y libgraphviz-dev | |
| shell: bash | |
| - name: Install R | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y r-base | |
| shell: bash | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' # Use Python 3.10 | |
| - name: Install dependencies | |
| run: | | |
| echo "Installing Dependencies" >> $GITHUB_OUTPUT | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| pip install jupyter nbconvert flake8 flake8-nb ipython | |
| shell: bash | |
| - name: Find changed notebooks | |
| id: find_notebooks | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }} | |
| git diff --name-only origin/${{ github.event.pull_request.base.ref }}...origin/${{ github.event.pull_request.head.ref }} > changed_files.txt | |
| grep '\.ipynb$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt | |
| echo "message=Found the following changed notebooks" >> $GITHUB_OUTPUT | |
| cat "changed_notebooks.txt" | |
| changed=$(cat "changed_notebooks.txt") | |
| echo "changed=$changed" >> $GITHUB_OUTPUT | |
| - name: Check if any notebooks changed | |
| id: check_notebooks | |
| run: | | |
| if grep -q '\.ipynb$' changed_notebooks.txt; then | |
| echo "notebooks_changed=true" >> $GITHUB_ENV | |
| else | |
| echo "notebooks_changed=false" >> $GITHUB_ENV | |
| fi | |
| - name: No notebooks changed | |
| if: env.notebooks_changed == 'false' | |
| run: echo "message=No Jupyter notebooks changed in this PR." >> $GITHUB_OUTPUT | |
| - name: Run Flake8 linting on changed notebooks | |
| if: env.notebooks_changed == 'true' | |
| id: lint | |
| run: | | |
| mkdir -p linting_logs | |
| flake8_errors=0 | |
| while IFS= read -r notebook; do | |
| [ "$(basename "$notebook")" = "python-dosearch.ipynb" ] && continue | |
| echo "notebook=Linting-$notebook" >> $GITHUB_OUTPUT | |
| flake8-nb --config=.flake8 "$notebook" > "linting_logs/$(basename "$notebook" .ipynb)_linting.txt" 2>&1 || flake8_errors=$((flake8_errors+1)) | |
| cat "linting_logs/$(basename "$notebook" .ipynb)_linting.txt" | |
| lint_errors = $(cat "linting_logs/$(basename "$notebook" .ipynb)_linting.txt") | |
| echo "lint_errors=$lint_errors" >> $GITHUB_OUTPUT | |
| done < changed_notebooks.txt | |
| echo "flake8_errors=$flake8_errors" >> $GITHUB_ENV | |
| echo "flake8_errors=$flake8_errors" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Convert changed Jupyter notebooks to Python scripts | |
| if: "! (matrix.folder == 'CM3' && (matrix.os == 'windows-latest' || matrix.os == 'macos-latest'))" | |
| id: convert | |
| run: | | |
| mkdir -p converted_scripts | |
| while IFS= read -r notebook; do | |
| echo "Processing $notebook" | |
| [ -e "$notebook" ] || continue | |
| [ "$(basename "$notebook")" = "DoubleML_and_Feature_Engineering_with_BERT.ipynb" ] && continue | |
| jupyter nbconvert --to script "$notebook" --output-dir converted_scripts | |
| done < changed_notebooks.txt | |
| shell: bash | |
| - name: Run converted Python scripts with IPython | |
| id: execute | |
| run: | | |
| mkdir -p logs | |
| script_errors=0 | |
| for script in converted_scripts/*.py; do | |
| [ -e "$script" ] || continue | |
| echo "Running $script" | |
| echo "script=$script" >> $GITHUB_OUTPUT | |
| ipython "$script" > "logs/$(basename "$script" .py).txt" 2>&1 || script_errors=$((script_errors+1)) | |
| done | |
| echo "script_errors=$script_errors" >> $GITHUB_ENV | |
| echo "script_errors=$script_errors" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Aggregate and report errors and warnings | |
| run: | | |
| echo "Aggregating errors and warnings..." | |
| grep -E "Traceback|Error:|Exception:|ModuleNotFoundError|FutureWarning|TypeError" logs/*.txt linting_logs/*.txt > logs/errors_and_warnings.txt || true | |
| echo "Errors and Warnings:" | |
| cat logs/errors_and_warnings.txt | |
| exec_errors=$(cat logs/errors_and_warnings.txt) | |
| echo "exec_errors=$exec_errors" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Upload linting logs | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: linting-logs-${{ matrix.folder }}-${{ matrix.os }} | |
| path: linting_logs | |
| - name: Upload execution logs | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: execution-logs-${{ matrix.folder }}-${{ matrix.os }} | |
| path: logs | |
| - name: Check for errors | |
| if: "(env.flake8_errors != '0' || env.script_errors != '0')" | |
| run: exit 1 |