Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add database #89

Merged
merged 9 commits into from
Aug 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ COMPOSE_PROJECT_NAME=ocrd_kitodo
MANAGER_IMAGE=ghcr.io/slub/ocrd_manager:latest # name and tag of image
MANAGER_HOST=ocrd-manager # name/address of server (for Kitodo)
MANAGER_PORT_SSH=9022 # host-side port to exposed SSH server (for external Kitodo)
MANAGER_PORT_WEB=4004 # port of web endpoint
MANAGER_ENV_UID=1001 # user id of SSH user (`id -u` when using `make`)
MANAGER_ENV_GID=1001 # group id of SSH user (`id -g` when using `make`)
MANAGER_ENV_UMASK=0002 # ssh user specific permission mask
Expand All @@ -17,6 +18,7 @@ MONITOR_IMAGE=ghcr.io/slub/ocrd_monitor:latest # name and tag of image
MONITOR_HOST=ocrd-monitor # name/address of server
MONITOR_PORT_WEB=5000 # host-side port to exposed Web server
MONITOR_PORT_LOG=8088 # host-side port to exposed Dozzle (Docker log viewer)
MONITOR_PORT_DBM=8082 # host-side port to exposed OCR-D Database Management (Mongo Express)

# Controller SSH server (with-ocrd-controller)
CONTROLLER_BASE_VERSION=maximum-cuda # tag of ocrd/all from stage version
Expand All @@ -35,8 +37,8 @@ CONTROLLER_WORKERS=1 # number of workers for processing
# Application Kitodo.Production (with-kitodo-production)
APP_IMAGE=ghcr.io/slub/ocrd_kitodo/kitodo_production_ocrd:latest # name and tag of image
APP_BUILD_CONTEXT=${PWD}/_modules/kitodo-production-docker/ # directory of Dockerfile
APP_BUILDER_GIT_COMMIT=ocrd-main # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`)
APP_BUILDER_GIT_SOURCE_URL=https://github.com/markusweigelt/kitodo-production/ # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`)
APP_BUILDER_GIT_REF=ocrd-main # branch "ocrd-main" of git repository (cause using `git` as `BUILDER_TYPE`)
APP_BUILDER_GIT_REPOSITORY=markusweigelt/kitodo-production # repository of `BUILDER_GIT_COMMIT` (cause using `git` as `BUILDER_TYPE`)
APP_DATA=${PWD}/kitodo/data # persistent volume of application data to mount, e.g. config and modules
APP_KEY=${PWD}/kitodo/.ssh/id_rsa # file path with private SSH key of `ocrd` user (should match one of `MANAGER_KEYS`)
APP_PORT=8080 # host-side port of Kitodo.Production
Expand All @@ -45,6 +47,7 @@ APP_PORT=8080 # host-side port of Kitodo.Production
DB_IMAGE=mysql:8.0.32 # name and tag of image
DB_HOST=kitodo-db # name/address of database (for Kitodo)
DB_PORT=3306 # host-side port to exposed database
DB_ROOT_USER=root
DB_ROOT_PASSWORD=1234 # root user password
DB_NAME=kitodo # name of database used by Kitodo.Production
DB_USER=kitodo # username to access database
Expand Down
2 changes: 1 addition & 1 deletion _modules/ocrd_manager
2 changes: 1 addition & 1 deletion _modules/ocrd_monitor
Submodule ocrd_monitor updated 71 files
+44 −0 .devcontainer/Dockerfile
+23 −0 .devcontainer/devcontainer.json
+454 −0 .devcontainer/library-scripts/common-debian.sh
+405 −0 .devcontainer/library-scripts/docker-in-docker-debian.sh
+14 −4 .github/workflows/test-ci.yml
+2 −1 .gitignore
+6 −1 Makefile
+17 −0 docker-browse-ocrd/Dockerfile
+5 −0 docker-browse-ocrd/init.sh
+29 −0 docker-compose.yml
+2 −2 init.sh
+5 −15 ocrdbrowser/__init__.py
+4 −68 ocrdbrowser/_browser.py
+10 −3 ocrdbrowser/_client.py
+69 −31 ocrdbrowser/_docker.py
+26 −21 ocrdbrowser/_port.py
+121 −49 ocrdbrowser/_subprocess.py
+9 −0 ocrdmonitor/database/__init__.py
+108 −0 ocrdmonitor/database/_browserprocessrepository.py
+52 −0 ocrdmonitor/database/_initdb.py
+41 −0 ocrdmonitor/database/_ocrdjobrepository.py
+45 −0 ocrdmonitor/environment.py
+3 −1 ocrdmonitor/main.py
+3 −45 ocrdmonitor/ocrdcontroller.py
+0 −88 ocrdmonitor/ocrdjob.py
+98 −0 ocrdmonitor/protocols.py
+26 −21 ocrdmonitor/server/app.py
+13 −6 ocrdmonitor/server/jobs.py
+34 −0 ocrdmonitor/server/lifespan.py
+3 −2 ocrdmonitor/server/logs.py
+0 −37 ocrdmonitor/server/proxy.py
+0 −107 ocrdmonitor/server/redirect.py
+67 −31 ocrdmonitor/server/settings.py
+6 −7 ocrdmonitor/server/templates/jobs.html.j2
+0 −109 ocrdmonitor/server/workspaces.py
+38 −0 ocrdmonitor/server/workspaces/__init__.py
+66 −0 ocrdmonitor/server/workspaces/_browsercommunication.py
+49 −0 ocrdmonitor/server/workspaces/_launchroutes.py
+23 −0 ocrdmonitor/server/workspaces/_listroutes.py
+126 −0 ocrdmonitor/server/workspaces/_proxyroutes.py
+948 −640 pdm.lock
+12 −10 pyproject.toml
+14 −0 tests/decorators.py
+24 −0 tests/markers.py
+109 −0 tests/ocrdbrowser/test_browser_launch.py
+0 −51 tests/ocrdbrowser/test_launch.py
+1 −1 tests/ocrdmonitor/conftest.py
+5 −1 tests/ocrdmonitor/server/conftest.py
+23 −0 tests/ocrdmonitor/server/decorators.py
+0 −60 tests/ocrdmonitor/server/fixtures.py
+0 −0 tests/ocrdmonitor/server/fixtures/__init__.py
+167 −0 tests/ocrdmonitor/server/fixtures/environment.py
+40 −0 tests/ocrdmonitor/server/fixtures/fixtureconfig.py
+29 −0 tests/ocrdmonitor/server/fixtures/repository.py
+26 −0 tests/ocrdmonitor/server/fixtures/settings.py
+63 −74 tests/ocrdmonitor/server/test_job_endpoint.py
+12 −28 tests/ocrdmonitor/server/test_settings.py
+22 −0 tests/ocrdmonitor/server/test_startup.py
+176 −99 tests/ocrdmonitor/server/test_workspace_endpoint.py
+0 −83 tests/ocrdmonitor/test_jobs.py
+0 −73 tests/ocrdmonitor/test_redirect.py
+2 −1 tests/ocrdmonitor/test_sshremote.py
+24 −4 tests/testdoubles/__init__.py
+7 −0 tests/testdoubles/_backgroundprocess.py
+1 −1 tests/testdoubles/_broadwayfake.py
+12 −26 tests/testdoubles/_browserfactory.py
+3 −0 tests/testdoubles/_browserfake.py
+54 −8 tests/testdoubles/_browserspy.py
+88 −0 tests/testdoubles/_inmemoryrepositories.py
+54 −0 tests/testdoubles/_registrybrowserfactory.py
+25 −0 tests/workspaces/a_workspace/mets.xml
25 changes: 22 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,34 @@ services:
service: ocrd-manager
depends_on:
- ocrd-controller
- ocrd-database

ocrd-monitor:
extends:
file: _modules/ocrd_monitor/docker-compose.yml
service: ocrd-monitor
depends_on:
- ocrd-controller
- ocrd-database

ocrd-logview:
extends:
file: _modules/ocrd_monitor/docker-compose.yml
service: ocrd-logview

ocrd-database:
extends:
file: _modules/ocrd_monitor/docker-compose.yml
service: ocrd-database

ocrd-database-management:
extends:
file: _modules/ocrd_monitor/docker-compose.yml
service: ocrd-database-management
depends_on:
ocrd-database:
condition: service_started

# OCR-D Controller

ocrd-controller:
Expand All @@ -46,11 +61,14 @@ services:
build:
args:
BUILDER_TYPE: git
BUILDER_GIT_COMMIT: ${APP_BUILDER_GIT_COMMIT}
BUILDER_GIT_SOURCE_URL: ${APP_BUILDER_GIT_SOURCE_URL}
BUILDER_GIT_REF: ${APP_BUILDER_GIT_REF}
BUILDER_GIT_REPOSITORY: ${APP_BUILDER_GIT_REPOSITORY}

environment:
- "OCRD_MANAGER=${MANAGER_HOST}:22"
APP_FOOTER_INFO: "Repository: <a href=\"https://github.com/${APP_BUILDER_GIT_REPOSITORY}/\">${APP_BUILDER_GIT_REPOSITORY}</a>, Ref: <a href=\"https://github.com/${APP_BUILDER_GIT_REPOSITORY}/tree/${APP_BUILDER_GIT_REF}\">${APP_BUILDER_GIT_REF}</a>"
GIT_REF: ${APP_BUILDER_GIT_REF}
GIT_REPOSITORY: ${APP_BUILDER_GIT_REPOSITORY}
OCRD_MANAGER: "${MANAGER_HOST}:22"

depends_on:
- kitodo-db
Expand Down Expand Up @@ -96,4 +114,5 @@ services:
- with-kitodo-production

volumes:
db-volume:
shared: