diff --git a/.env b/.env index 1144471c..f2cb72b6 100644 --- a/.env +++ b/.env @@ -20,10 +20,10 @@ MONITOR_PORT_WEB=5000 # host-side port to exposed Web server MONITOR_PORT_LOG=8088 # host-side port to exposed Dozzle (Docker log viewer) MONITOR_PORT_DBM=8082 # host-side port to exposed OCR-D Database Management (Mongo Express) MONITOR_DB_ROOT_USER=root # root user of OCR-D database -MONITOR_DB_ROOT_USER=root_password # root password of OCR-D database +MONITOR_DB_ROOT_PASSWORD=root_password # root password of OCR-D database # Controller SSH server (with-ocrd-controller) -CONTROLLER_BASE_VERSION=maximum-cuda # tag of ocrd/all from stage version +CONTROLLER_BASE_VERSION=2024-03-08 # tag of ocrd/all from stage version CONTROLLER_IMAGE=ghcr.io/slub/ocrd_controller:latest # name and tag of image CONTROLLER_HOST=ocrd-controller # name/address of server (for Manager/Monitor) CONTROLLER_PORT_SSH=22 # SSH port (for Manager/Monitor) @@ -39,8 +39,8 @@ CONTROLLER_WORKERS=1 # number of workers for processing # Application Kitodo.Production (with-kitodo-production) APP_IMAGE=ghcr.io/slub/ocrd_kitodo/kitodo_production_ocrd:latest # name and tag of image APP_BUILD_CONTEXT=${PWD}/_modules/kitodo-production-docker/ # directory of Dockerfile -APP_BUILDER_GIT_REF=ocrd-main # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`) -APP_BUILDER_GIT_REPOSITORY=markusweigelt/kitodo-production # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`) +APP_BUILDER_GIT_REF=master # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`) +APP_BUILDER_GIT_REPOSITORY=kitodo/kitodo-production # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`) APP_DATA=${PWD}/kitodo/data # persistent volume of application data to mount, e.g. config and modules APP_KEY=${PWD}/kitodo/.ssh/id_rsa # file path with private SSH key of `ocrd` user (should match one of `MANAGER_KEYS`) APP_PORT=8080 # host-side port of Kitodo.Production @@ -65,4 +65,3 @@ ES_NODE_PORT=9300 # host-side port to inter-node protocol MQ_IMAGE=ghcr.io/slub/kitodo-production-activemq:latest # name and tag of image MQ_HOST=kitodo-mq # name/address of server (for Kitodo/Manager) MQ_PORT=61616 # host-side port to exposed TCP Transport -MQ_CLIENT_QUEUE=TaskActionQueue diff --git a/.github/workflows/makefile-ci.yml b/.github/workflows/makefile-ci.yml index 59095d73..6b0d27e0 100644 --- a/.github/workflows/makefile-ci.yml +++ b/.github/workflows/makefile-ci.yml @@ -39,7 +39,7 @@ jobs: sudo systemctl restart docker - name: Checkout repository and submodules - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive @@ -74,8 +74,25 @@ jobs: - name: Test parallel using multiple workers run: CONTROLLER_WORKERS=3 make -j test + + - name: Collect docker logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + dest: './docker-logs' + + - name: Tar docker logs on failure + if: failure() + run: tar cvzf ./docker-logs.tgz ./docker-logs - - name: Setup upterm session when failure + - name: Upload docker logs to GitHub + if: failure() + uses: actions/upload-artifact@master + with: + name: docker-logs.tgz + path: ./docker-logs.tgz + + - name: Setup upterm session on failure if: ${{ failure() && github.event_name == 'workflow_dispatch' }} uses: lhotari/action-upterm@v1 @@ -106,7 +123,7 @@ jobs: sudo systemctl restart docker - name: Checkout repository and submodules - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive @@ -163,13 +180,30 @@ jobs: run: make start - name: Test - run: make test + run: make test - name: Clean testdata run: make clean-testdata - name: Test parallel using multiple workers run: CONTROLLER_WORKERS=3 make -j test + + - name: Collect docker logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + with: + dest: './docker-logs' + + - name: Tar docker logs on failure + if: failure() + run: tar cvzf ./docker-logs.tgz ./docker-logs + + - name: Upload docker logs to GitHub + if: failure() + uses: actions/upload-artifact@master + with: + name: docker-logs.tgz + path: ./docker-logs.tgz - name: Setup upterm session when failure if: ${{ failure() && github.event_name == 'workflow_dispatch' }} diff --git a/Makefile b/Makefile index f8a9d40c..ee9ddd50 100644 --- a/Makefile +++ b/Makefile @@ -139,13 +139,17 @@ $(CONTROLLER_MODELS)/ocrd-resources/ocrd-tesserocr-recognize/frak2021.traineddat wget -O $@ https://ub-backup.bib.uni-mannheim.de/~stweil/tesstrain/frak2021/tessdata_best/frak2021-0.905.traineddata build: - docker compose build + docker compose build $(SERVICES) --no-cache + +rebuild: + docker compose stop $(SERVICES) + docker compose up -d --build $(SERVICES) start: docker compose up -d down: - docker compose down + docker compose down -v stop: docker compose stop @@ -177,8 +181,9 @@ test-kitodo: $(APP_DATA)/metadata/testdata-kitodo rm -rf $(APP_DATA)/metadata/testdata-kitodo/ocr # wait until Kitodo.Production directory structure is initialized docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 kitodo-app:$$APP_PORT" + docker exec -t $(APP_CONTAINER) bash -c "/wait-for-it.sh -t 0 ocrd-database:27017" # run asynchronous ocr processing, which should return within 5 seconds with exit status 1 - timeout --preserve-status 5 docker exec -t $(APP_CONTAINER) bash -c '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id "testdata-kitodo" --task-id 1'; test $$? = 1 + timeout --preserve-status 5 docker exec $(APP_CONTAINER) bash -c '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id "testdata-kitodo" --task-id 1'; test $$? = 1 # check with interval of 1 second if ocr folder exists. It fails if the ocr folder is not created within 5 minutes. timeout 10m bash -c 'until test -s $(APP_DATA)/metadata/testdata-kitodo/ocr/alto/00000014.tif.original.xml; do sleep 5; done' # rest if the alto directory and file exist diff --git a/_modules/ocrd_controller b/_modules/ocrd_controller index b5896667..6f7bcf7d 160000 --- a/_modules/ocrd_controller +++ b/_modules/ocrd_controller @@ -1 +1 @@ -Subproject commit b589666768a01f0347a395221fa966f6c25d748f +Subproject commit 6f7bcf7d3ff573401dbd2a6ed755dc3bbad464f8 diff --git a/_modules/ocrd_manager b/_modules/ocrd_manager index d6cf314e..3461853d 160000 --- a/_modules/ocrd_manager +++ b/_modules/ocrd_manager @@ -1 +1 @@ -Subproject commit d6cf314edbfc2df44382036683edd4b5404e2501 +Subproject commit 3461853de28408e9ed0baac799f6ce7614b18b7d diff --git a/_modules/ocrd_monitor b/_modules/ocrd_monitor index 9eb7b1ae..f2f09753 160000 --- a/_modules/ocrd_monitor +++ b/_modules/ocrd_monitor @@ -1 +1 @@ -Subproject commit 9eb7b1aebd59f9098a74fa4ce6b23657ff497273 +Subproject commit f2f097531dd9322f6e1684a88328677d05a0d4bc diff --git a/_resources/kitodo-sample/data/dms-export/.gitkeep b/_resources/kitodo-sample/data/export/.gitkeep similarity index 100% rename from _resources/kitodo-sample/data/dms-export/.gitkeep rename to _resources/kitodo-sample/data/export/.gitkeep diff --git a/_resources/kitodo-sample/sql/kitodo_post_init.sql b/_resources/kitodo-sample/sql/kitodo_post_init.sql index 61cb3b61..459b169b 100644 --- a/_resources/kitodo-sample/sql/kitodo_post_init.sql +++ b/_resources/kitodo-sample/sql/kitodo_post_init.sql @@ -1,15 +1,12 @@ -INSERT INTO `ocrprofile` (`id`, `title`, `file`, `client_id`) VALUES - (1, 'OCR Workflow Example', '/workflows/ocr-workflow-default.sh', 1); - INSERT INTO `ruleset` (`id`, `title`, `file`, `orderMetadataByRuleset`, `indexAction`, `active`, `client_id`) VALUES (4, 'Ruleset', 'ruleset_default.xml', 0, 'INDEX', 1, 1); INSERT INTO `workflow` (`id`, `title`, `status`, `indexAction`, `client_id`, `separateStructure`) VALUES (2, 'OCR_Workflow', 'ACTIVE', 'INDEX', 1, 0); -INSERT INTO `template` (`id`, `title`, `creationDate`, `sortHelperStatus`, `ruleset_id`, `docket_id`, `indexAction`, `workflow_id`, `active`, `client_id`, `ocr_profile_id`) VALUES - (1, 'Example Template', '2021-11-22 17:52:48', NULL, 4, 1, 'INDEX', 1, 1, 1, 1), - (2, 'Template_OCR', '2022-03-10 16:42:15', NULL, 4, 1, 'INDEX', 2, 1, 1, 1); +INSERT INTO `template` (`id`, `title`, `creationDate`, `sortHelperStatus`, `ruleset_id`, `docket_id`, `indexAction`, `workflow_id`, `active`, `client_id`, `ocrd_workflow_id` ) VALUES + (1, 'Example Template', '2021-11-22 17:52:48', NULL, 4, 1, 'INDEX', 1, 1, 1, NULL), + (2, 'Template_OCR', '2022-03-10 16:42:15', NULL, 4, 1, 'INDEX', 2, 1, 1, 'ocr-workflow-default.sh'); INSERT INTO `process` (`id`, `title`, `inChoiceListShown`, `sortHelperStatus`, `sortHelperImages`, `sortHelperArticles`, `sortHelperDocstructs`, `sortHelperMetadata`, `creationDate`, `wikiField`, `project_id`, `ruleset_id`, `docket_id`, `indexAction`, `processBaseUri`, `template_id`, `parent_id`, `ordering`, `exported`) VALUES (3, 'CaelCaFeD_1761630636', 0, '060020000020', 0, 0, 0, 0, '2022-03-10 16:43:28', '', 1, 4, 1, 'INDEX', '3/', 2, NULL, NULL, 0); @@ -43,13 +40,13 @@ INSERT INTO `task` (`id`, `title`, `ordering`, `processingStatus`, `editType`, ` (8, 'Export DMS', 4, 0, 0, NULL, NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 1, NULL, 'Task_4', 0, 1, 0, 0, 0, 0), (18, 'Scanning', 1, 1, 4, '2022-03-10 16:42:51', NULL, NULL, 0, 0, 0, 1, 1, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_1', 0, 0, 0, 0, 0, 0), (19, 'QC', 2, 0, 0, NULL, NULL, NULL, 0, 0, 0, 1, 0, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_2', 0, 0, 0, 0, 0, 0), - (20, 'OCR from Process Dir', 3, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrprofilefile} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0), + (20, 'OCR from Process Dir', 3, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrdworkflowid} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0), (21, 'Structure and Metadata', 4, 0, 0, NULL, NULL, NULL, 0, 1, 0, 0, 0, 0, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_3', 0, 0, 0, 0, 0, 0), (22, 'Export DMS', 5, 0, 0, NULL, NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Task_4', 0, 0, 0, 0, 0, 0), (23, 'OCR from Export Dir', 6, 0, 0, NULL, NULL, NULL, 0, 0, 1, 0, 0, 0, 0, 'OCR Export Dir', '/usr/local/kitodo/scripts/script_ocr_export_dir.sh {processtitle} {stepid}', 0, 0, NULL, NULL, 'INDEX', 2, NULL, 'Activity_12zha6z', 0, 1, 0, 0, 0, 0), (24, 'Scanning', 1, 3, 3, NOW(), NULL, NULL, 0, 0, 0, 1, 1, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_1', 0, 0, 0, 0, 0, 0), (25, 'QC', 2, 3, 3, NOW(), NULL, NULL, 0, 0, 0, 1, 0, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_2', 0, 0, 0, 0, 0, 0), - (26, 'OCR from Process Dir', 3, 1, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrprofilefile} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0), + (26, 'OCR from Process Dir', 3, 1, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, 'OCR Process Dir', '/usr/local/kitodo/scripts/script_ocr_process_dir.sh --proc-id {processid} --task-id {stepid} --workflow {ocrdworkflowid} --lang $(meta.topstruct.DocLanguage) --script $(meta.topstruct.slub_script)', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_0y2ldc1', 0, 0, 0, 0, 0, 0), (27, 'Structure and Metadata', 4, 0, 3, NOW(), NULL, NULL, 0, 1, 0, 0, 0, 0, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_3', 0, 0, 0, 0, 0, 0), (28, 'Export DMS', 5, 0, 3, NOW(), NULL, NULL, 0, 0, 0, 0, 0, 1, 0, NULL, NULL, 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Task_4', 0, 0, 0, 0, 0, 0), (29, 'OCR from Export Dir', 6, 0, 3, NOW(), NULL, NULL, 0, 0, 1, 0, 0, 0, 0, 'OCR Export Dir', '/usr/local/kitodo/scripts/script_ocr_export_dir.sh {processtitle} {stepid}', 0, 0, NULL, 3, 'INDEX', NULL, NULL, 'Activity_12zha6z', 0, 1, 0, 0, 0, 0); @@ -166,17 +163,16 @@ INSERT INTO `role_x_authority` (`role_id`, `authority_id`) VALUES (1, 189), (1, 190), (1, 191), - (1, 192), - (1, 193), - (1, 194), - (1, 195), - (1, 196), - (1, 197), - (1, 198), - (1, 199), - (1, 200), - (1, 201), - (1, 202); + (1, 192), + (1, 193), + (1, 194), + (1, 195), + (1, 196), + (1, 197), + (1, 198), + (1, 199), + (1, 200), + (1, 201); INSERT INTO `task_x_role` (`role_id`, `task_id`) VALUES (1, 5), @@ -228,4 +224,4 @@ INSERT INTO `workpiece_x_property` (`process_id`, `property_id`) VALUES (3, 39), (3, 40); -UPDATE project SET dmsImportRootPath = '/usr/local/kitodo/dms-export/' WHERE id = 1; +UPDATE project SET dmsImportRootPath = '/usr/local/kitodo/export/' WHERE id = 1; diff --git a/_resources/kitodo/before_startup.sh b/_resources/kitodo/before_startup.sh index 3043bb82..60cee25f 100755 --- a/_resources/kitodo/before_startup.sh +++ b/_resources/kitodo/before_startup.sh @@ -26,10 +26,10 @@ if [ -n "$OCRD_MANAGER" ]; then ssh-keyscan -H -p ${OCRD_MANAGER_PORT:-22} $OCRD_MANAGER_HOST,$OCRD_MANAGER_IP >>/etc/ssh/ssh_known_hosts fi -# link OCR-D Manager workflow into OCR profiles folder -if [ ! -d "/tmp/kitodo/overwrites/data/ocr/profiles" ]; then - mkdir -p /tmp/kitodo/overwrites/data/ocr/profiles/ +# link OCR-D Manager workflow into OCR-D workflows directory +if [ ! -d "/tmp/kitodo/overwrites/data/ocrd_workflows" ]; then mkdir -p /usr/local/ocrd-manager/workflows/ - ln -s /usr/local/ocrd-manager/workflows /tmp/kitodo/overwrites/data/ocr/profiles/workflows + ln -s /usr/local/ocrd-manager/workflows/ /tmp/kitodo/overwrites/data/ + mv /tmp/kitodo/overwrites/data/workflows /tmp/kitodo/overwrites/data/ocrd_workflows fi diff --git a/_resources/kitodo/data/scripts/script_ocr_process_dir.sh b/_resources/kitodo/data/scripts/script_ocr_process_dir.sh index 4258e39f..a07e409a 100755 --- a/_resources/kitodo/data/scripts/script_ocr_process_dir.sh +++ b/_resources/kitodo/data/scripts/script_ocr_process_dir.sh @@ -66,6 +66,9 @@ if test -n "$WORKFLOW"; then COMMAND+=" --workflow $WORKFLOW" fi +if test -n "$MQ_HOST" -a -n "$MQ_PORT"; then + COMMAND+=" --activemq-url tcp://$MQ_HOST:$MQ_PORT?closeAsync=false --activemq-queue TaskActionQueue --async" +fi COMMAND+=" /data/$PROCESS_ID" diff --git a/docker-compose.yml b/docker-compose.yml index 1af93abe..2d089d1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,6 @@ services: file: _modules/ocrd_monitor/docker-compose.yml service: ocrd-monitor depends_on: - - ocrd-controller - ocrd-database ocrd-logview: