reworking EVERYTHING #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/deploy.yml | |
| name: Task1 QC and Deploy | |
| on: | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: write | |
| packages: write | |
| issues: write | |
| id-token: write | |
| pages: write | |
| jobs: | |
| process_raw: | |
| name: Process Raw CSV Files | |
| runs-on: self-hosted | |
| outputs: | |
| sub: ${{ steps.set_vars.outputs.sub }} | |
| task: ${{ steps.set_vars.outputs.task }} | |
| version: ${{ steps.set_vars.outputs.version }} | |
| run_part: ${{ steps.set_vars.outputs.run_part }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Get Changed CSV Files | |
| run: | | |
| #!/bin/bash | |
| # Get the list of CSV files changed in the last 24 hours | |
| data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq) | |
| # Export the data variable to the environment | |
| echo "data=$data" >> $GITHUB_ENV | |
| # Print the changed CSV files | |
| echo "Changed CSV files in the last 24 hours: $data" | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Parse Raw CSV Files | |
| id: set_vars | |
| run: | | |
| # Loop through each CSV file in $data | |
| for file in $data; do | |
| # Extract the directory and filename | |
| dir=$(dirname "$file") | |
| filename=$(basename "$file") | |
| # Extract the run-* part from the directory | |
| run_part=$(basename "$dir") | |
| # Split the filename into sub, task, and version | |
| IFS='_' read -r sub task version <<< "$filename" | |
| version="${version%.csv}" # Remove the .csv extension from version | |
| # Set outputs | |
| echo "::set-output name=run_part::$run_part" | |
| echo "::set-output name=sub::$sub" | |
| echo "::set-output name=task::$task" | |
| echo "::set-output name=version::$version" | |
| # Print the extracted values | |
| echo "Run Part: $run_part" | |
| echo "Subject: $sub" | |
| echo "Task: $task" | |
| echo "Version: $version" | |
| done | |
| run_qc: | |
| name: Run Quality Control | |
| runs-on: self-hosted | |
| needs: process_raw | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Debug Environment Variables | |
| run: | | |
| echo "Subject: ${{ needs.process_raw.outputs.sub }}" | |
| echo "Task: ${{ needs.process_raw.outputs.task }}" | |
| echo "Version: ${{ needs.process_raw.outputs.version }}" | |
| echo "Run Part: ${{ needs.process_raw.outputs.run_part }}" | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| - name: Run Quality Control Script | |
| run: | | |
| sub=${{ needs.process_raw.outputs.sub }} | |
| task=${{ needs.process_raw.outputs.task }} | |
| vers=${{ needs.process_raw.outputs.version }} | |
| run_part=${{ needs.process_raw.outputs.run_part }} | |
| for sub in ${sub}; do | |
| echo "Processing subject: $sub" | |
| for task in ${task}; do | |
| echo "Processing task: $task" | |
| for vers in ${vers}; do | |
| echo "Processing version: $vers" | |
| csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv" | |
| mkdir -p "./data/${sub}/${run_part}" | |
| log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log" | |
| echo "CSV file: $csv_file" | |
| echo "Log file: $log_file" | |
| if [ -f "$csv_file" ]; then | |
| python ./code/VNBqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file" | |
| echo "QC for ${sub}_${task}_${vers} completed" | |
| else | |
| echo "CSV file $csv_file does not exist" | |
| fi | |
| done | |
| done | |
| done | |
| - name: Commit and Push QC Results | |
| run: | | |
| git config --global user.name "miloswrath" | |
| git config --global user.email "[email protected]" | |
| git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
| git add ./data/*/*/*.png | |
| git commit -m "Add QC results for subjects $(date +%Y-%m-%d)" | |
| git push origin main | |
| - name: List Directory After QC | |
| run: | | |
| echo "Listing directory after running QC:" | |
| find ./data -type d | |
| find ./data -type f | |
| add: | |
| name: Generate Jekyll Posts and Deploy | |
| runs-on: ubuntu-latest | |
| needs: [process_raw, run_qc] | |
| steps: | |
| # 1. Checkout the Repository | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false # Recommended for security | |
| fetch-depth: 0 | |
| ref: main | |
| - name: Pull Latest Changes | |
| run: git pull origin main | |
| # 4. Set Up Ruby Environment | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.1' # Specify your Ruby version | |
| bundler-cache: true # Caches installed gems automatically | |
| cache-version: 1 # Increment if you need to reset the cache | |
| # 5. Install Ruby Dependencies | |
| - name: Install Dependencies | |
| run: bundle install | |
| # 6. Generate Jekyll Posts from PNGs | |
| - name: Generate Jekyll Posts | |
| run: | | |
| set -e # Exit immediately if a command exits with a non-zero status | |
| set -x # Print commands and their arguments as they are executed | |
| run_part=${{ needs.process_raw.outputs.run_part }} | |
| POSTS_DIR="_posts" | |
| mkdir -p "$POSTS_DIR" # Ensure the _posts directory exists | |
| # Initialize an associative array to group images by subject | |
| declare -A subjects | |
| # Get the list of PNG files added in the latest commit | |
| images=$(git diff --name-only HEAD~1 HEAD | grep '\.png$') | |
| echo "Images: $images" | |
| # Iterate over each PNG file and group them by subject number | |
| for file in $images; do | |
| # Check if the file exists to avoid errors | |
| if [ ! -f "$file" ]; then | |
| continue | |
| fi | |
| # Extract the subject number from the filename (assuming it's the first part before '_') | |
| filename=$(basename "$file") | |
| subject=$(echo "$filename" | awk -F_ '{print $1}') | |
| # Append the filename to the subject's array | |
| subjects["$subject"]+="$file " | |
| done | |
| # Generate Jekyll posts for each subject | |
| for subject in "${!subjects[@]}"; do | |
| # Define the post filename with current date and subject number | |
| timestamp=$(date +%H%M%S) | |
| post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md" | |
| # Create the Jekyll post | |
| { | |
| echo "---" | |
| echo "layout: post" | |
| echo "title: Subject $subject" | |
| echo "date: $(date +%Y-%m-%d)" | |
| echo "categories: subjects" | |
| echo "---" | |
| echo "" | |
| # Add images to the post | |
| for image in ${subjects["$subject"]}; do | |
| # Adjust the image path based on your site structure | |
| # Assuming images are served from the root | |
| echo "" | |
| done | |
| } > "$post_filename" | |
| echo "Created post: $post_filename" | |
| done | |
| # 7. List _posts Directory for Verification (Optional) | |
| - name: List _posts Directory | |
| run: | | |
| echo "Listing _posts directory:" | |
| ls -la _posts | |
| # 8. Commit and Push Generated Posts | |
| - name: Commit and Push Posts | |
| run: | | |
| git config --global user.name "miloswrath" | |
| git config --global user.email "[email protected]" | |
| git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
| # Add new posts to git | |
| git add _posts/*.md | |
| # Commit changes if there are any | |
| if ! git diff --cached --exit-code > /dev/null; then | |
| git commit -m "Add new posts for subjects $(date +%Y-%m-%d)" | |
| git push origin main # Replace 'main' with your default branch if different | |
| else | |
| echo "No changes to commit." | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # 9. Build the Jekyll Site | |
| - name: Build with Jekyll | |
| run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}" | |
| env: | |
| JEKYLL_ENV: production | |
| # 10. Deploy to GitHub Pages | |
| # Using GitHub's built-in Pages action | |
| - name: Configure GitHub Pages | |
| uses: actions/configure-pages@v5 | |
| - name: Upload Pages Artifact | |
| uses: actions/upload-pages-artifact@v1 | |
| with: | |
| path: ./_site # Ensure this matches your Jekyll build output | |
| - name: List _site Directory | |
| run: | | |
| echo "Listing _site directory:" | |
| ls -la _site | |
| - name: Deploy to GitHub Pages | |
| uses: actions/deploy-pages@v1 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} |