From 714db54a26ec9a932e7b8df11776bbff28649996 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 11:10:47 +0200 Subject: [PATCH 01/13] copying assetstore from dev-5 to instance --- .github/actions/import-db/action.yml | 9 ++++++++- .github/workflows/deploy.yml | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index ead62157438..ad2d5b3a20d 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -10,6 +10,10 @@ inputs: description: 'port suffix' required: true type: string + ASSETSTORE: + description: 'location of assetstore folder' + required: false + type: string runs: using: "composite" @@ -49,6 +53,9 @@ runs: echo "=====" docker logs $DB5NAME || true echo "=====" + # copy assetstore + cp -r $ASSETSTORE dspace${INSTANCE}:/dspace/ + echo "=====" cd ../ pip install -r requirements.txt || true echo "=====" @@ -62,4 +69,4 @@ runs: run: | docker stop ${{ steps.import.outputs.cid }} || true if: ${{ always() }} - + diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4da6d06a5bc..589724d1c00 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -103,6 +103,7 @@ jobs: with: INSTANCE: ${{ env.INSTANCE }} DATADIR: /opt/dspace-data/clarin-dspace/ + ASSETSTORE: /opt/dspace-data/clarin-dspace/assetstore - name: dspace basic command run: | @@ -140,7 +141,7 @@ jobs: run: | # wait until FE stabilizes a bit sleep 3m - + curl -H "Accept: application/vnd.github.everest-preview+json" \ -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \ --request POST \ @@ -211,7 +212,7 @@ jobs: run: | # wait until FE stabilizes a bit sleep 3m - + curl -H "Accept: application/vnd.github.everest-preview+json" \ -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" \ --request POST \ From a0606cdb41ee3cfc49e632c3470f6067cd9cd82d Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 11:47:02 +0200 Subject: [PATCH 02/13] removed -r flag --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index ad2d5b3a20d..ea61fa4e674 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -54,7 +54,7 @@ runs: docker logs $DB5NAME || true echo "=====" # copy assetstore - cp -r $ASSETSTORE dspace${INSTANCE}:/dspace/ + cp $ASSETSTORE dspace${INSTANCE}:/dspace/ echo "=====" cd ../ pip install -r requirements.txt || true From 1b2b08e9baf9ae6979fa380fa73a602a919f8fe4 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 11:56:03 +0200 Subject: [PATCH 03/13] redefination of input --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index ea61fa4e674..1706d99c745 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -54,7 +54,7 @@ runs: docker logs $DB5NAME || true echo "=====" # copy assetstore - cp $ASSETSTORE dspace${INSTANCE}:/dspace/ + cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" cd ../ pip install -r requirements.txt || true From 57312bfe7a9a9a7fb0044fbd1d725d33b670788d Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 12:06:32 +0200 Subject: [PATCH 04/13] added -r flag --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 1706d99c745..2fae2aa6afc 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -54,7 +54,7 @@ runs: docker logs $DB5NAME || true echo "=====" # copy assetstore - cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ + cp -r ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" cd ../ pip install -r requirements.txt || true From 3a5abe65d9b26249ae3af9a56a94320c3013268c Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 12:21:21 +0200 Subject: [PATCH 05/13] replaced cp by docker cp --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 2fae2aa6afc..1ecc6257a5a 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -54,7 +54,7 @@ runs: docker logs $DB5NAME || true echo "=====" # copy assetstore - cp -r ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ + docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" cd ../ pip install -r requirements.txt || true From c4cc77ac7aefcb6f6c7b14aa53892b1bc4908f42 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 13:46:59 +0200 Subject: [PATCH 06/13] removed assetstore and add path to python --- .github/actions/import-db/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 1ecc6257a5a..7763562781b 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -54,6 +54,10 @@ runs: docker logs $DB5NAME || true echo "=====" # copy assetstore + echo Preparing assetstore + # remove old assetstore + docker exec dspace${{ inputs.INSTANCE }} rm -r /dspace/ + # copy new assetstore docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" cd ../ @@ -62,7 +66,7 @@ runs: cd ./src # cleanup resume rm __temp/resume/*.json || true - python3 repo_import.py --resume=false --config=backend.endpoint=$BEURL --config=db_dspace_7.port=$DB7PORT --config=db_dspace_5.port=$DB5PORT --config=db_utilities_5.port=$DB5PORT --config=input.datadir=$DATADIR/data/ --config=input.icondir=$DATADIR/icon/ + python3 repo_import.py --resume=false --config=backend.endpoint=$BEURL --config=db_dspace_7.port=$DB7PORT --config=db_dspace_5.port=$DB5PORT --config=db_utilities_5.port=$DB5PORT --config=input.datadir=$DATADIR/data/ --config=input.icondir=$DATADIR/icon/ --assetstore=${{ inputs.ASSETSTORE }} - name: cleanup shell: bash From 506d79cb977e41c87cf1c9e2e372e60eab6a0a19 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 14:19:39 +0200 Subject: [PATCH 07/13] TEST: assesttore --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 7763562781b..36085c8c869 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -27,7 +27,7 @@ runs: - uses: actions/checkout@v4 with: repository: dataquest-dev/dspace-import - ref: 'main' + ref: 'assetstore-test' submodules: 'recursive' path: 'dspace-import' From 29b376d813b19ee3781ba4df64ba99c7f6fcbe26 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 14:28:32 +0200 Subject: [PATCH 08/13] removed only assetstore not all from dspace --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 36085c8c869..cb19056b9f7 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -56,7 +56,7 @@ runs: # copy assetstore echo Preparing assetstore # remove old assetstore - docker exec dspace${{ inputs.INSTANCE }} rm -r /dspace/ + docker exec dspace${{ inputs.INSTANCE }} rm -r /dspace/assetstore # copy new assetstore docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" From 4c4992dd56df20d0b8e8ee29165dd5340314b6a9 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Thu, 20 Jun 2024 14:39:40 +0200 Subject: [PATCH 09/13] removed removing assetstore - it is part of erase database, not import --- .github/actions/import-db/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index cb19056b9f7..3073130266d 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -55,9 +55,6 @@ runs: echo "=====" # copy assetstore echo Preparing assetstore - # remove old assetstore - docker exec dspace${{ inputs.INSTANCE }} rm -r /dspace/assetstore - # copy new assetstore docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ echo "=====" cd ../ From 499d6480fea6ea7dc804018de51508548583f31e Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 21 Jun 2024 07:12:04 +0200 Subject: [PATCH 10/13] run import on the main branch --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 3073130266d..b953bd0dadb 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -27,7 +27,7 @@ runs: - uses: actions/checkout@v4 with: repository: dataquest-dev/dspace-import - ref: 'assetstore-test' + ref: 'main' submodules: 'recursive' path: 'dspace-import' From 7d37f8f597512cf7011237cf2d67272eea1081a2 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 21 Jun 2024 11:58:56 +0200 Subject: [PATCH 11/13] empty assetstore control --- .github/actions/import-db/action.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index b953bd0dadb..9120d1b1e30 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -55,7 +55,11 @@ runs: echo "=====" # copy assetstore echo Preparing assetstore - docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ + if [[ -z ${{ inputs.ASSETSTORE }} ]]; then + echo Location of assetstore folder is empty. Not copping assetstore + else + docker cp ${{ inputs.ASSETSTORE }} dspace${{ inputs.INSTANCE }}:/dspace/ + fi echo "=====" cd ../ pip install -r requirements.txt || true @@ -63,8 +67,22 @@ runs: cd ./src # cleanup resume rm __temp/resume/*.json || true - python3 repo_import.py --resume=false --config=backend.endpoint=$BEURL --config=db_dspace_7.port=$DB7PORT --config=db_dspace_5.port=$DB5PORT --config=db_utilities_5.port=$DB5PORT --config=input.datadir=$DATADIR/data/ --config=input.icondir=$DATADIR/icon/ --assetstore=${{ inputs.ASSETSTORE }} - + # arguments of the script + # required + args=( + --resume=false + --config=backend.endpoint=$BEURL + --config=db_dspace_7.port=$DB7PORT + --config=db_dspace_5.port=$DB5PORT + --config=db_utilities_5.port=$DB5PORT + --config=input.datadir=$DATADIR/data/ + --config=input.icondir=$DATADIR/icon/ + ) + # Add --resume argument if inputs.ASSETSTORE is defined + if [ -n "${{ inputs.ASSETSTORE }}" ]; then + args+=(--assetstore=${{ inputs.ASSETSTORE }}) + fi + python3 repo_import.py "${args[@]}" - name: cleanup shell: bash run: | From d08cf6d22066c32664bee25081760263b4b7bb80 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Fri, 21 Jun 2024 12:38:28 +0200 Subject: [PATCH 12/13] test assetstore folder --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 9120d1b1e30..72a388f3e00 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -27,7 +27,7 @@ runs: - uses: actions/checkout@v4 with: repository: dataquest-dev/dspace-import - ref: 'main' + ref: 'assetstore-test' submodules: 'recursive' path: 'dspace-import' From 8ea58720f1e1c1befc159a96b2db228c92b9c1f3 Mon Sep 17 00:00:00 2001 From: Paurikova2 Date: Mon, 24 Jun 2024 06:08:12 +0200 Subject: [PATCH 13/13] correct branch after test --- .github/actions/import-db/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/import-db/action.yml b/.github/actions/import-db/action.yml index 72a388f3e00..9120d1b1e30 100644 --- a/.github/actions/import-db/action.yml +++ b/.github/actions/import-db/action.yml @@ -27,7 +27,7 @@ runs: - uses: actions/checkout@v4 with: repository: dataquest-dev/dspace-import - ref: 'assetstore-test' + ref: 'main' submodules: 'recursive' path: 'dspace-import'