Merge pull request #98 from slub/adjustments-webhook

Update Kitodo.Production and scripts for webhook manager version

.env

@@ -20,10 +20,10 @@ MONITOR_PORT_WEB=5000 # host-side port to exposed Web server
 MONITOR_PORT_LOG=8088 # host-side port to exposed Dozzle (Docker log viewer)
 MONITOR_PORT_DBM=8082 # host-side port to exposed OCR-D Database Management (Mongo Express)
 MONITOR_DB_ROOT_USER=root # root user of OCR-D database
-MONITOR_DB_ROOT_USER=root_password # root password of OCR-D database
+MONITOR_DB_ROOT_PASSWORD=root_password # root password of OCR-D database
 
 # Controller SSH server (with-ocrd-controller)
-CONTROLLER_BASE_VERSION=maximum-cuda # tag of ocrd/all from stage version
+CONTROLLER_BASE_VERSION=2024-03-08 # tag of ocrd/all from stage version
 CONTROLLER_IMAGE=ghcr.io/slub/ocrd_controller:latest # name and tag of image
 CONTROLLER_HOST=ocrd-controller # name/address of server (for Manager/Monitor)
 CONTROLLER_PORT_SSH=22 # SSH port (for Manager/Monitor)
@@ -39,8 +39,8 @@ CONTROLLER_WORKERS=1 # number of workers for processing
 # Application Kitodo.Production (with-kitodo-production)
 APP_IMAGE=ghcr.io/slub/ocrd_kitodo/kitodo_production_ocrd:latest # name and tag of image
 APP_BUILD_CONTEXT=${PWD}/_modules/kitodo-production-docker/ # directory of Dockerfile
-APP_BUILDER_GIT_REF=ocrd-main # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`)
-APP_BUILDER_GIT_REPOSITORY=markusweigelt/kitodo-production # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`)
+APP_BUILDER_GIT_REF=master # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`)
+APP_BUILDER_GIT_REPOSITORY=kitodo/kitodo-production # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`)
 APP_DATA=${PWD}/kitodo/data # persistent volume of application data to mount, e.g. config and modules
 APP_KEY=${PWD}/kitodo/.ssh/id_rsa # file path with private SSH key of `ocrd` user (should match one of `MANAGER_KEYS`)
 APP_PORT=8080 # host-side port of Kitodo.Production
@@ -65,4 +65,3 @@ ES_NODE_PORT=9300 # host-side port to inter-node protocol
 MQ_IMAGE=ghcr.io/slub/kitodo-production-activemq:latest # name and tag of image
 MQ_HOST=kitodo-mq # name/address of server (for Kitodo/Manager)
 MQ_PORT=61616 # host-side port to exposed TCP Transport
-MQ_CLIENT_QUEUE=TaskActionQueue

.github/workflows/makefile-ci.yml

@@ -39,7 +39,7 @@ jobs:
         sudo systemctl restart docker
     
     - name: Checkout repository and submodules
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -74,8 +74,25 @@ jobs:
 
     - name: Test parallel using multiple workers 
       run: CONTROLLER_WORKERS=3 make -j test 
+
+    - name: Collect docker logs on failure
+      if: failure()
+      uses: jwalton/gh-docker-logs@v2
+      with:
+        dest: './docker-logs'
+
+    - name: Tar docker logs on failure
+      if: failure()
+      run: tar cvzf ./docker-logs.tgz ./docker-logs
 
-    - name: Setup upterm session when failure
+    - name: Upload docker logs to GitHub
+      if: failure()
+      uses: actions/upload-artifact@master
+      with:
+        name: docker-logs.tgz
+        path: ./docker-logs.tgz
+
+    - name: Setup upterm session on failure
       if: ${{ failure() && github.event_name == 'workflow_dispatch' }}
       uses: lhotari/action-upterm@v1  
 
@@ -106,7 +123,7 @@ jobs:
         sudo systemctl restart docker
     
     - name: Checkout repository and submodules
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -163,13 +180,30 @@ jobs:
       run: make start
 
     - name: Test
-      run: make test 
+      run: make test
 
     - name: Clean testdata
       run: make clean-testdata
 
     - name: Test parallel using multiple workers 
       run: CONTROLLER_WORKERS=3 make -j test
+
+    - name: Collect docker logs on failure
+      if: failure()
+      uses: jwalton/gh-docker-logs@v2
+      with:
+        dest: './docker-logs'
+
+    - name: Tar docker logs on failure
+      if: failure()
+      run: tar cvzf ./docker-logs.tgz ./docker-logs
+
+    - name: Upload docker logs to GitHub
+      if: failure()
+      uses: actions/upload-artifact@master
+      with:
+        name: docker-logs.tgz
+        path: ./docker-logs.tgz
 
     - name: Setup upterm session when failure
       if: ${{ failure() && github.event_name == 'workflow_dispatch' }}

Makefile

@@ -139,13 +139,17 @@ $(CONTROLLER_MODELS)/ocrd-resources/ocrd-tesserocr-recognize/frak2021.traineddat
 	wget -O $@ https://ub-backup.bib.uni-mannheim.de/~stweil/tesstrain/frak2021/tessdata_best/frak2021-0.905.traineddata
 
 build:
-	docker compose build
+	docker compose build $(SERVICES) --no-cache
+
+rebuild:
+	docker compose stop $(SERVICES)
+	docker compose up -d --build $(SERVICES)
 
 start:
 	docker compose up -d
 
 down:
-	docker compose down
+	docker compose down -v 
 
 stop:
 	docker compose stop
@@ -177,8 +181,9 @@ test-kitodo: $(APP_DATA)/metadata/testdata-kitodo
 	rm -rf $(APP_DATA)/metadata/testdata-kitodo/ocr
 # wait until Kitodo.Production directory structure is initialized
 	docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 kitodo-app:$$APP_PORT"
+	docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 ocrd-database:27017"
 # run asynchronous ocr processing, which should return within 5 seconds with exit status 1
-	timeout --preserve-status 5 docker exec -t $(APP_CONTAINER) bash -c '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id "testdata-kitodo" --task-id 1'; test $$? = 1
+	timeout --preserve-status 5 docker exec $(APP_CONTAINER) bash -c '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id "testdata-kitodo" --task-id 1'; test $$? = 1
 # check with interval of 1 second if ocr folder exists. It fails if the ocr folder is not created within 5 minutes.
 	timeout 10m bash -c 'until test -s $(APP_DATA)/metadata/testdata-kitodo/ocr/alto/00000014.tif.original.xml; do sleep 5; done'
 # rest if the alto directory and file exist

_resources/kitodo-sample/sql/kitodo_post_init.sql

@@ -1,15 +1,12 @@
-INSERT INTO `ocrprofile` (`id`, `title`, `file`, `client_id`) VALUES
-	(1, 'OCR Workflow Example', '/workflows/ocr-workflow-default.sh', 1);
-
 INSERT INTO `ruleset` (`id`, `title`, `file`, `orderMetadataByRuleset`, `indexAction`, `active`, `client_id`) VALUES
 	(4, 'Ruleset', 'ruleset_default.xml', 0, 'INDEX', 1, 1);
 
 INSERT INTO `workflow` (`id`, `title`, `status`, `indexAction`, `client_id`, `separateStructure`) VALUES
 	(2, 'OCR_Workflow', 'ACTIVE', 'INDEX', 1, 0);
 
-INSERT INTO `template` (`id`, `title`, `creationDate`, `sortHelperStatus`, `ruleset_id`, `docket_id`, `indexAction`, `workflow_id`, `active`, `client_id`, `ocr_profile_id`) VALUES
- 	(1, 'Example Template', '2021-11-22 17:52:48', NULL, 4, 1, 'INDEX', 1, 1, 1, 1),
-	(2, 'Template_OCR', '2022-03-10 16:42:15', NULL, 4, 1, 'INDEX', 2, 1, 1, 1);
+INSERT INTO `template` (`id`, `title`, `creationDate`, `sortHelperStatus`, `ruleset_id`, `docket_id`, `indexAction`, `workflow_id`, `active`, `client_id`, `ocrd_workflow_id` ) VALUES
+ 	(1, 'Example Template', '2021-11-22 17:52:48', NULL, 4, 1, 'INDEX', 1, 1, 1, NULL),
+	(2, 'Template_OCR', '2022-03-10 16:42:15', NULL, 4, 1, 'INDEX', 2, 1, 1, 'ocr-workflow-default.sh');
 
 INSERT INTO `process` (`id`, `title`, `inChoiceListShown`, `sortHelperStatus`, `sortHelperImages`, `sortHelperArticles`, `sortHelperDocstructs`, `sortHelperMetadata`, `creationDate`, `wikiField`, `project_id`, `ruleset_id`, `docket_id`, `indexAction`, `processBaseUri`, `template_id`, `parent_id`, `ordering`, `exported`) VALUES
 	(3, 'CaelCaFeD_1761630636', 0, '060020000020', 0, 0, 0, 0, '2022-03-10 16:43:28', '', 1, 4, 1, 'INDEX', '3/', 2, NULL, NULL, 0);
@@ -43,13 +40,13 @@ INSERT INTO `task` (`id`, `title`, `ordering`, `processingStatus`, `editType`, `
 	(8, 'Export DMS', 4, 0, 0, NULL, NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 1, NULL, 'Task_4', 0, 1, 0, 0, 0, 0),
 	(18, 'Scanning', 1, 1, 4, '2022-03-10 16:42:51', NULL, NULL, 0, 0, 0, 1, 1, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_1', 0, 0, 0, 0, 0, 0),
 	(19, 'QC', 2, 0, 0, NULL, NULL, NULL, 0, 0, 0, 1, 0, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_2', 0, 0, 0, 0, 0, 0),
-    (20, 'OCR from Process Dir', 3, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrprofilefile} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0),
+    (20, 'OCR from Process Dir', 3, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrdworkflowid} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0),
 	(21, 'Structure and Metadata', 4, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_3', 0, 0, 0, 0, 0, 0),
 	(22, 'Export DMS', 5, 0, 0, NULL, NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_4', 0, 0, 0, 0, 0, 0),
     (23, 'OCR from Export Dir', 6, 0, 0, NULL, NULL, NULL, 0, 0, 1, 0, 0, 0, 0, 'OCR Export Dir', '/usr/local/kitodo/scripts/script_ocr_export_dir.sh {processtitle} {stepid}', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_12zha6z', 0, 1, 0, 0, 0, 0),
 	(24, 'Scanning', 1, 3, 3, NOW(), NULL, NULL, 0, 0, 0, 1, 1, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_1', 0, 0, 0, 0, 0, 0),
 	(25, 'QC', 2, 3, 3, NOW(), NULL, NULL, 0, 0, 0, 1, 0, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_2', 0, 0, 0, 0, 0, 0),
-    (26, 'OCR from Process Dir', 3, 1, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrprofilefile} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0),
+    (26, 'OCR from Process Dir', 3, 1, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrdworkflowid} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0),
 	(27, 'Structure and Metadata', 4, 0, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_3', 0, 0, 0, 0, 0, 0),
 	(28, 'Export DMS', 5, 0, 3, NOW(), NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_4', 0, 0, 0, 0, 0, 0),
     (29, 'OCR from Export Dir', 6, 0, 3, NOW(), NULL, NULL, 0, 0, 1, 0, 0, 0, 0, 'OCR Export Dir', '/usr/local/kitodo/scripts/script_ocr_export_dir.sh {processtitle} {stepid}', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_12zha6z', 0, 1, 0, 0, 0, 0);
@@ -166,17 +163,16 @@ INSERT INTO `role_x_authority` (`role_id`, `authority_id`) VALUES
 	(1, 189),
 	(1, 190),
 	(1, 191),
-    (1, 192),
-    (1, 193),
-    (1, 194),
-    (1, 195),
-    (1, 196),
-    (1, 197),
-    (1, 198),
-    (1, 199),
-    (1, 200),
-    (1, 201),
-	(1, 202);
+	(1, 192),
+	(1, 193),
+	(1, 194),
+	(1, 195),
+	(1, 196),
+	(1, 197),
+	(1, 198),
+	(1, 199),
+	(1, 200),
+	(1, 201);
 
 INSERT INTO `task_x_role` (`role_id`, `task_id`) VALUES
 	(1, 5),
@@ -228,4 +224,4 @@ INSERT INTO `workpiece_x_property` (`process_id`, `property_id`) VALUES
 	(3, 39),
 	(3, 40);
 
-UPDATE project SET dmsImportRootPath = '/usr/local/kitodo/dms-export/' WHERE id = 1;
+UPDATE project SET dmsImportRootPath = '/usr/local/kitodo/export/' WHERE id = 1;

_resources/kitodo/before_startup.sh

@@ -26,10 +26,10 @@ if [ -n "$OCRD_MANAGER" ]; then
   ssh-keyscan -H -p ${OCRD_MANAGER_PORT:-22} $OCRD_MANAGER_HOST,$OCRD_MANAGER_IP >>/etc/ssh/ssh_known_hosts
 fi
 
-# link OCR-D Manager workflow into OCR profiles folder
-if [ ! -d "/tmp/kitodo/overwrites/data/ocr/profiles" ]; then
-  mkdir -p /tmp/kitodo/overwrites/data/ocr/profiles/
+# link OCR-D Manager workflow into OCR-D workflows directory
+if [ ! -d "/tmp/kitodo/overwrites/data/ocrd_workflows" ]; then
   mkdir -p /usr/local/ocrd-manager/workflows/
 
-  ln -s /usr/local/ocrd-manager/workflows /tmp/kitodo/overwrites/data/ocr/profiles/workflows
+  ln -s /usr/local/ocrd-manager/workflows/ /tmp/kitodo/overwrites/data/
+  mv /tmp/kitodo/overwrites/data/workflows /tmp/kitodo/overwrites/data/ocrd_workflows
 fi

_resources/kitodo/data/scripts/script_ocr_process_dir.sh

@@ -66,6 +66,9 @@ if test -n "$WORKFLOW"; then
     COMMAND+=" --workflow $WORKFLOW"
 fi
 
+if test -n "$MQ_HOST" -a -n "$MQ_PORT"; then
+    COMMAND+=" --activemq-url tcp://$MQ_HOST:$MQ_PORT?closeAsync=false --activemq-queue TaskActionQueue --async"
+fi
 
 COMMAND+=" /data/$PROCESS_ID"
 

docker-compose.yml

@@ -15,7 +15,6 @@ services:
       file: _modules/ocrd_monitor/docker-compose.yml
       service: ocrd-monitor
     depends_on:
-      - ocrd-controller
       - ocrd-database
 
   ocrd-logview: