Skip to content

Commit

Permalink
test
Browse files Browse the repository at this point in the history
  • Loading branch information
@jianlins
jianlins authored Dec 20, 2024
1 parent a47a179 commit 9d10ecb
Showing 1 changed file with 163 additions and 163 deletions.
326 changes: 163 additions & 163 deletions .github/workflows/build_win_env.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,192 +61,192 @@ jobs:
create_env:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
# - name: Set up Python
# uses: actions/setup-python@v5
# with:
# python-version: '3.10'

- name: Check Setting Folder
run: |
pwd
ls ${{ github.event.inputs.build_folder }}
shell: bash
# - name: Check Setting Folder
# run: |
# pwd
# ls ${{ github.event.inputs.build_folder }}
# shell: bash

- name: Cache Miniforge Environment
uses: actions/cache@v4
with:
path: ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
key: ${{ runner.os }}-${{ github.event.inputs.build_folder }}-${{ hashFiles('${{ github.event.inputs.build_folder }}/environment.yml') }}
restore-keys: |
${{ runner.os }}-${{ github.event.inputs.build_folder }}-
# - name: Cache Miniforge Environment
# uses: actions/cache@v4
# with:
# path: ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# key: ${{ runner.os }}-${{ github.event.inputs.build_folder }}-${{ hashFiles('${{ github.event.inputs.build_folder }}/environment.yml') }}
# restore-keys: |
# ${{ runner.os }}-${{ github.event.inputs.build_folder }}-


- name: run pre build script
shell: pwsh {0}
run: |
cd ${{ github.event.inputs.build_folder }}
pwd
if (Test-Path -Path ".\pre_build.ps1") {
# sometimes the activate above doesn't work for the following ps1
pwsh .\pre_build.ps1 ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
}else{
pwd
ls
Write-Output ".\pre_build.ps1 doesn't exist. Skip executing it."
}
# - name: run pre build script
# shell: pwsh {0}
# run: |
# cd ${{ github.event.inputs.build_folder }}
# pwd
# if (Test-Path -Path ".\pre_build.ps1") {
# # sometimes the activate above doesn't work for the following ps1
# pwsh .\pre_build.ps1 ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
# }else{
# pwd
# ls
# Write-Output ".\pre_build.ps1 doesn't exist. Skip executing it."
# }

- name: Install miniforge
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-version: latest
activate-environment: ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
environment-file: ${{ github.event.inputs.build_folder }}/environment.yml
auto-activate-base: false
# - name: Install miniforge
# uses: conda-incubator/setup-miniconda@v3
# with:
# miniforge-version: latest
# activate-environment: ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# environment-file: ${{ github.event.inputs.build_folder }}/environment.yml
# auto-activate-base: false


- name: Create Conda Environment
shell: pwsh
run: |
cd ${{ github.event.inputs.build_folder }}
$envPath = "${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}"
# Check if the environment directory exists
if (Test-Path -Path $envPath) {
Write-Host "Environment ${{ github.event.inputs.build_folder }} already exists under ${{ github.event.inputs.target_folder }}. Will try update..."
# conda env update -f environment.yml -p ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
} else {
# Environment doesn't exist, proceed to create a new one
Write-Host "Creating new environment ${{ github.event.inputs.build_folder }} under ${{ github.event.inputs.target_folder }}."
# conda env create -f environment.yml -p ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
}
echo "clean up cache..."
conda clean --all -f -y
pip cache purge
# - name: Create Conda Environment
# shell: pwsh
# run: |
# cd ${{ github.event.inputs.build_folder }}
# $envPath = "${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}"
# # Check if the environment directory exists
# if (Test-Path -Path $envPath) {
# Write-Host "Environment ${{ github.event.inputs.build_folder }} already exists under ${{ github.event.inputs.target_folder }}. Will try update..."
# # conda env update -f environment.yml -p ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# } else {
# # Environment doesn't exist, proceed to create a new one
# Write-Host "Creating new environment ${{ github.event.inputs.build_folder }} under ${{ github.event.inputs.target_folder }}."
# # conda env create -f environment.yml -p ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# }
# echo "clean up cache..."
# conda clean --all -f -y
# pip cache purge

- name: check final environment settings
run: |
conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
conda env export
# - name: check final environment settings
# run: |
# conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# conda env export

- name: Set HADOOP_HOME environment variable
if: ${{ github.event.inputs.download_jars == 'true' }}
run: |
New-Item -ItemType Directory -Force -Path "C:\hadoop\bin"
echo "HADOOP_HOME=C:\hadoop" | Out-File -FilePath $env:GITHUB_ENV -Append
echo "C:\hadoop\bin" | Out-File -FilePath $env:GITHUB_PATH -Append
# - name: Set HADOOP_HOME environment variable
# if: ${{ github.event.inputs.download_jars == 'true' }}
# run: |
# New-Item -ItemType Directory -Force -Path "C:\hadoop\bin"
# echo "HADOOP_HOME=C:\hadoop" | Out-File -FilePath $env:GITHUB_ENV -Append
# echo "C:\hadoop\bin" | Out-File -FilePath $env:GITHUB_PATH -Append

- name: Download Hadoop DLLs for Windows
if: ${{ github.event.inputs.download_jars == 'true' }}
env:
PYSPARK_JARS_IVY: ${{ github.event.inputs.target_folder }}/ivy
run: |
Invoke-WebRequest -Uri "https://github.com/steveloughran/winutils/raw/master/hadoop-3.0.0/bin/winutils.exe" -OutFile "C:\hadoop\bin\winutils.exe"
# Assuming hadoop.dll is also required and available at a certain URL - replace this URL with the actual location for hadoop.dll
Invoke-WebRequest -Uri "https://github.com/steveloughran/winutils/raw/master/hadoop-3.0.0/bin/hadoop.dll" -OutFile "C:\hadoop\bin\hadoop.dll"
echo "Create .ivy folder"
New-Item -ItemType Directory -Force -Path ${{ env.PYSPARK_JARS_IVY }}/jars
# - name: Download Hadoop DLLs for Windows
# if: ${{ github.event.inputs.download_jars == 'true' }}
# env:
# PYSPARK_JARS_IVY: ${{ github.event.inputs.target_folder }}/ivy
# run: |
# Invoke-WebRequest -Uri "https://github.com/steveloughran/winutils/raw/master/hadoop-3.0.0/bin/winutils.exe" -OutFile "C:\hadoop\bin\winutils.exe"
# # Assuming hadoop.dll is also required and available at a certain URL - replace this URL with the actual location for hadoop.dll
# Invoke-WebRequest -Uri "https://github.com/steveloughran/winutils/raw/master/hadoop-3.0.0/bin/hadoop.dll" -OutFile "C:\hadoop\bin\hadoop.dll"
# echo "Create .ivy folder"
# New-Item -ItemType Directory -Force -Path ${{ env.PYSPARK_JARS_IVY }}/jars

# change the default .ivy location to ${{ github.event.inputs.target_folder }}/ivy so that can be zipped together easily
- name: Cache Ivy jars
if: ${{ github.event.inputs.download_jars == 'true' }}
uses: actions/cache@v4
with:
path: |
$ivyDir = ${{ github.event.inputs.target_folder }}/ivy/jars
key: ${{ runner.os }}-ivy-${{ hashFiles('**/*.jar') }}
restore-keys: |
${{ runner.os }}-ivy-
# # change the default .ivy location to ${{ github.event.inputs.target_folder }}/ivy so that can be zipped together easily
# - name: Cache Ivy jars
# if: ${{ github.event.inputs.download_jars == 'true' }}
# uses: actions/cache@v4
# with:
# path: |
# $ivyDir = ${{ github.event.inputs.target_folder }}/ivy/jars
# key: ${{ runner.os }}-ivy-${{ hashFiles('**/*.jar') }}
# restore-keys: |
# ${{ runner.os }}-ivy-

- name: download sparknlp jars
if: ${{ github.event.inputs.download_jars == 'true' }}
env:
PYSPARK_JARS_IVY: ${{ github.event.inputs.target_folder }}/ivy
run: |
echo "PYSPARK_JARS_IVY=${{ env.PYSPARK_JARS_IVY }}"
conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# if (!(Test-Path -Path "${{ env.PYSPARK_JARS_IVY }}/jars/spark-nlp-gpu-assembly-${{ github.event.inputs.sparknlp_version }}.jar")) {
# curl -Lo "${{ env.PYSPARK_JARS_IVY }}/jars/spark-nlp${{ github.event.inputs.sparknlp_gpu }}-assembly-${{ github.event.inputs.sparknlp_version }}.jar" https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/jars/spark-nlp${{ github.event.inputs.sparknlp_gpu }}-assembly-${{ github.event.inputs.sparknlp_version }}.jar
# } else {
# echo "JAR already exists in ${{ env.PYSPARK_JARS_IVY }}/jars"
# }
if ( [string]::IsNullOrEmpty('${{ github.event.inputs.sparknlp_gpu }}')){
python -c "import sparknlp; spark = sparknlp.start(params={'spark.jars.ivy':'${{ github.event.inputs.target_folder }}/ivy'}); spark.stop();"
}else{
python -c "import sparknlp; spark = sparknlp.start(gpu=True, params={'spark.jars.ivy':'${{ github.event.inputs.target_folder }}/ivy'}); spark.stop();"
}
# - name: download sparknlp jars
# if: ${{ github.event.inputs.download_jars == 'true' }}
# env:
# PYSPARK_JARS_IVY: ${{ github.event.inputs.target_folder }}/ivy
# run: |
# echo "PYSPARK_JARS_IVY=${{ env.PYSPARK_JARS_IVY }}"
# conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# # if (!(Test-Path -Path "${{ env.PYSPARK_JARS_IVY }}/jars/spark-nlp-gpu-assembly-${{ github.event.inputs.sparknlp_version }}.jar")) {
# # curl -Lo "${{ env.PYSPARK_JARS_IVY }}/jars/spark-nlp${{ github.event.inputs.sparknlp_gpu }}-assembly-${{ github.event.inputs.sparknlp_version }}.jar" https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/jars/spark-nlp${{ github.event.inputs.sparknlp_gpu }}-assembly-${{ github.event.inputs.sparknlp_version }}.jar
# # } else {
# # echo "JAR already exists in ${{ env.PYSPARK_JARS_IVY }}/jars"
# # }
# if ( [string]::IsNullOrEmpty('${{ github.event.inputs.sparknlp_gpu }}')){
# python -c "import sparknlp; spark = sparknlp.start(params={'spark.jars.ivy':'${{ github.event.inputs.target_folder }}/ivy'}); spark.stop();"
# }else{
# python -c "import sparknlp; spark = sparknlp.start(gpu=True, params={'spark.jars.ivy':'${{ github.event.inputs.target_folder }}/ivy'}); spark.stop();"
# }

- name: download spacy models
if: ${{ github.event.inputs.download_spacy_model == 'true' }}
run: |
conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
python -m spacy download en_core_web_sm
python -m spacy download en_core_web_md
# - name: copy cached jars
# - name: download spacy models
# if: ${{ github.event.inputs.download_spacy_model == 'true' }}
# run: |
# $sourceDir = Join-Path -Path $env:USERPROFILE -ChildPath ".ivy"
# Get-ChildItem -Path $sourceDir -Recurse -Filter *.jar | Copy-Item -Destination ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}/lib/site-packages/pyspark/jars -Force
# conda activate ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# python -m spacy download en_core_web_sm
# python -m spacy download en_core_web_md

- name: check folder
run: |
ls ${{ github.event.inputs.target_folder }}
ls ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}
# # - name: copy cached jars
# # run: |
# # $sourceDir = Join-Path -Path $env:USERPROFILE -ChildPath ".ivy"
# # Get-ChildItem -Path $sourceDir -Recurse -Filter *.jar | Copy-Item -Destination ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}/lib/site-packages/pyspark/jars -Force

# - name: check folder
# run: |
# ls ${{ github.event.inputs.target_folder }}
# ls ${{ github.event.inputs.target_folder }}/${{ github.event.inputs.build_folder }}

- name: check jar folder
if: ${{ github.event.inputs.download_jars == 'true' }}
run: |
ls ${{ github.event.inputs.target_folder }}/ivy
ls ${{ github.event.inputs.target_folder }}/ivy/jars
# - name: check jar folder
# if: ${{ github.event.inputs.download_jars == 'true' }}
# run: |
# ls ${{ github.event.inputs.target_folder }}/ivy
# ls ${{ github.event.inputs.target_folder }}/ivy/jars

- name: clear jar cache folder
if: ${{ github.event.inputs.download_jars == 'true' && github.event.inputs.remove_cache == 'true' }}
run: |
Remove-Item -Path ${{ github.event.inputs.target_folder }}/ivy/cache/* -Recurse -Force
# - name: clear jar cache folder
# if: ${{ github.event.inputs.download_jars == 'true' && github.event.inputs.remove_cache == 'true' }}
# run: |
# Remove-Item -Path ${{ github.event.inputs.target_folder }}/ivy/cache/* -Recurse -Force

- name: run post build script
shell: pwsh {0}
run: |
cd ${{ github.event.inputs.build_folder }}
pwd
conda activate ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
if (Test-Path -Path ".\post_build.ps1") {
# sometimes the activate above doesn't work for the following ps1
pwsh .\post_build.ps1 ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
}else{
pwd
ls
Write-Output ".\post_build.ps1 doesn't exist. Skip executing it."
}
# - name: run post build script
# shell: pwsh {0}
# run: |
# cd ${{ github.event.inputs.build_folder }}
# pwd
# conda activate ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
# if (Test-Path -Path ".\post_build.ps1") {
# # sometimes the activate above doesn't work for the following ps1
# pwsh .\post_build.ps1 ${{ github.event.inputs.target_folder }}\${{ github.event.inputs.build_folder }}
# }else{
# pwd
# ls
# Write-Output ".\post_build.ps1 doesn't exist. Skip executing it."
# }

- name: Check for 7-Zip installation
run: |
if (!(Test-Path "C:\Program Files\7-Zip\7z.exe")) {
choco install 7zip
}
# - name: Check for 7-Zip installation
# run: |
# if (!(Test-Path "C:\Program Files\7-Zip\7z.exe")) {
# choco install 7zip
# }

- name: Compress and split folder
if: ${{ github.event.inputs.zip_jar_only == 'false' }}
run: |
pwd
7z a -t7z -v${{ github.event.inputs.zip_vol_size }} zipped/${{ github.event.inputs.build_folder }}.7z ${{ github.event.inputs.target_folder }}*
ls zipped
# - name: Compress and split folder
# if: ${{ github.event.inputs.zip_jar_only == 'false' }}
# run: |
# pwd
# 7z a -t7z -v${{ github.event.inputs.zip_vol_size }} zipped/${{ github.event.inputs.build_folder }}.7z ${{ github.event.inputs.target_folder }}*
# ls zipped

- name: Compress and split ivy folder
if: ${{ github.event.inputs.zip_jar_only == 'true' }}
run: |
pwd
7z a -t7z -v${{ github.event.inputs.zip_vol_size }} zipped/${{ github.event.inputs.build_folder }}_ivy.7z ${{ github.event.inputs.target_folder }}/ivy*
ls zipped
# - name: Compress and split ivy folder
# if: ${{ github.event.inputs.zip_jar_only == 'true' }}
# run: |
# pwd
# 7z a -t7z -v${{ github.event.inputs.zip_vol_size }} zipped/${{ github.event.inputs.build_folder }}_ivy.7z ${{ github.event.inputs.target_folder }}/ivy*
# ls zipped

- name: Upload compressed parts as artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ github.event.inputs.build_folder }}
path: zipped/*.7z.*
retention-days: ${{ github.event.inputs.retention_days }}
# - name: Upload compressed parts as artifacts
# uses: actions/upload-artifact@v4
# with:
# name: ${{ github.event.inputs.build_folder }}
# path: zipped/*.7z.*
# retention-days: ${{ github.event.inputs.retention_days }}

- name: Determine tag name
id: tag
Expand Down

0 comments on commit 9d10ecb

Please sign in to comment.