diff --git a/README.md b/README.md index 37de3cd..89f395c 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,11 @@ To run the server, please execute the following from the root directory: ```bash bash pip3 install -e . -cd server +cd src mkdir -p logs export DB_URL=postgresql+asyncpg://npg_rw:$PASS@npg_porch_db:$PORT/$DATABASE export DB_SCHEMA='non_default' -uvicorn npg.main:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json +uvicorn npg_porch.server:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json ``` and open your browser at `http://localhost:8080` to see links to the docs. @@ -52,7 +52,7 @@ The server will not start without `DB_URL` in the environment When you want HTTPS, logging and all that jazz: ```bash -uvicorn main:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt +uvicorn server:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt ``` Consider running with nohup or similar. @@ -62,7 +62,7 @@ Some notes on arguments: --host: 0.0.0.0 = bind to all network interfaces. Reliable but greedy in some situations ---log-config: Refers to a JSON file for python logging library. An example file is found in /server/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs. +--log-config: Refers to a JSON file for python logging library. An example file is found in /src/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs. --ssl-keyfile: A PEM format key for the server certificate --ssl-certfile: A PEM format certificate for signing HTTPS communications @@ -77,11 +77,11 @@ pip install -e .[test] pytest ``` -Individual tests are run in the form `pytest server/tests/init_test.py` +Individual tests are run in the form `pytest tests/init_test.py` ### Fixtures -Fixtures reside under `server/tests/fixtures` and are registered in `server/tests/conftest.py` +Fixtures reside under `tests/fixtures` and are registered in `tests/conftest.py` They can also be listed by invoking `pytest --fixtures` Any fixtures that are not imported in `conftest.py` will not be detected. @@ -106,7 +106,7 @@ The SET command ensures that the new schema is visible _for one session only_ in DB=npg_porch export DB_URL=postgresql+psycopg2://npg_admin:$PASS@npg_porch_db:$PORT/$DB # note that the script requires a regular PG driver, not the async version showed above -server/deploy_schema.py +src/deploy_schema.py psql --host=npg_porch_db --port=$PORT --username=npg_admin --password -d $DB ``` diff --git a/docs/user_guide.md b/docs/user_guide.md index 73e50ba..e677a3d 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -30,7 +30,7 @@ Access to the service is loosely controlled with authorisation tokens. You will ### Step 1 - register your pipeline with npg_porch -*Schema: npg.porch.model.pipeline* +*Schema: npg_porch.model.pipeline* Nothing in npg_porch can happen until there's a pipeline defined. For our purposes "pipeline" means "a thing you can run", and it may refer to specific code, or a wrapper that can run the pipeline in this particular way with some standard arguments. @@ -50,7 +50,7 @@ You can name your pipeline however you like, but the name must be unique, and be Keep this pipeline definition with your data, as you will need it to tell npg_porch which pipeline you are acting on. -When communicating with npg_porch (as with any HTTP server) you must inspect the response code and message after each communication. See `-w " %{http_code}" above. The API documentation lists the response codes you can expect to have to handle. In this case, the server may respond with 400 - BAD REQUEST if you leave out a name, or 409 - CONFLICT if you chose a name that is already created. +As with any HTTP server, when communicating with npg_porch you must inspect the response code and message after each communication. See `-w " %{http_code}" above. The API documentation lists the response codes you can expect to have to handle. In this case, the server may respond with 400 - BAD REQUEST if you leave out a name, or 409 - CONFLICT if you chose a name that is already created. ### Step 2 - decide on the unique criteria for running the pipeline @@ -58,7 +58,7 @@ e.g. Once per 24 hours, poll iRODS metadata for data relating to a study. We might create a cronjob that runs a script. It invokes `imeta` and retrieves a list of results. Now we turn each of those results into a JSON document to our own specification: -*Schema: npg.porch.model.task* +*Schema: npg_porch.model.task* **study-100-id-run-45925.json** @@ -113,7 +113,7 @@ Note that it is possible to run the same `task_input` with a different `pipeline ### Step 3 - register the documents with npg_porch -*Schema: npg.porch.model.task* +*Schema: npg_porch.model.task* Now you want the pipeline to run once per specification, and so register the documents with npg_porch. diff --git a/scripts/deploy_schema.py b/scripts/deploy_schema.py index 9f56d71..ea7adf4 100644 --- a/scripts/deploy_schema.py +++ b/scripts/deploy_schema.py @@ -3,7 +3,7 @@ import os import sqlalchemy -import npg.porchdb.models +import npg_porch.db.models db_url = os.environ.get('DB_URL') schema_name = os.environ.get('DB_SCHEMA') @@ -17,5 +17,5 @@ connect_args={'options': f'-csearch_path={schema_name}'} ) -npg.porchdb.models.Base.metadata.schema = schema_name -npg.porchdb.models.Base.metadata.create_all(engine) +npg_porch.db.models.Base.metadata.schema = schema_name +npg_porch.db.models.Base.metadata.create_all(engine) diff --git a/scripts/issue_token.py b/scripts/issue_token.py index b31e4db..6300f1a 100755 --- a/scripts/issue_token.py +++ b/scripts/issue_token.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.exc import NoResultFound -from npg.porchdb.models import Token, Pipeline +from npg_porch.db.models import Token, Pipeline parser = argparse.ArgumentParser( description='Creates a token in the backend DB and returns it' diff --git a/server/npg/porch/endpoints/__init__.py b/server/npg/porch/endpoints/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/npg/porchdb/__init__.py b/server/npg/porchdb/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9eb1408..0000000 --- a/setup.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[metadata] -name = npg_porch -version = 1.0.0 - -[options] -package_dir = - =server -packages = find: - -[options.packages.find] -where=server \ No newline at end of file diff --git a/server/logging.json b/src/logging.json similarity index 100% rename from server/logging.json rename to src/logging.json diff --git a/server/__init__.py b/src/npg_porch/__init__.py similarity index 100% rename from server/__init__.py rename to src/npg_porch/__init__.py diff --git a/server/npg/__init__.py b/src/npg_porch/auth/__init__.py similarity index 100% rename from server/npg/__init__.py rename to src/npg_porch/auth/__init__.py diff --git a/server/npg/porch/auth/token.py b/src/npg_porch/auth/token.py similarity index 91% rename from server/npg/porch/auth/token.py rename to src/npg_porch/auth/token.py index 49cf89c..0fa4ed9 100644 --- a/server/npg/porch/auth/token.py +++ b/src/npg_porch/auth/token.py @@ -23,8 +23,8 @@ from fastapi.security import HTTPBearer from fastapi import HTTPException -from npg.porchdb.connection import get_CredentialsValidator -from npg.porchdb.auth import CredentialsValidationException +from npg_porch.db.connection import get_CredentialsValidator +from npg_porch.db.auth import CredentialsValidationException auth_scheme = HTTPBearer() diff --git a/server/npg/porch/__init__.py b/src/npg_porch/db/__init__.py similarity index 100% rename from server/npg/porch/__init__.py rename to src/npg_porch/db/__init__.py diff --git a/server/npg/porchdb/auth.py b/src/npg_porch/db/auth.py similarity index 96% rename from server/npg/porchdb/auth.py rename to src/npg_porch/db/auth.py index 3d30ef5..364ee21 100644 --- a/server/npg/porchdb/auth.py +++ b/src/npg_porch/db/auth.py @@ -23,8 +23,8 @@ from sqlalchemy.orm import contains_eager from sqlalchemy.orm.exc import NoResultFound -from npg.porchdb.models import Token -from npg.porch.models.permission import Permission, RolesEnum +from npg_porch.db.models import Token +from npg_porch.models.permission import Permission, RolesEnum __AUTH_TOKEN_LENGTH__ = 32 __AUTH_TOKEN_REGEXP__ = re.compile( diff --git a/server/npg/porchdb/connection.py b/src/npg_porch/db/connection.py similarity index 90% rename from server/npg/porchdb/connection.py rename to src/npg_porch/db/connection.py index 1374db3..fea8c3e 100644 --- a/server/npg/porchdb/connection.py +++ b/src/npg_porch/db/connection.py @@ -22,9 +22,9 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from npg.porchdb.models import Base -from npg.porchdb.data_access import AsyncDbAccessor -from npg.porchdb.auth import Validator +from npg_porch.db.models import Base +from npg_porch.db.data_access import AsyncDbAccessor +from npg_porch.db.auth import Validator config = { 'DB_URL': os.environ.get('DB_URL'), @@ -34,7 +34,6 @@ if config['TEST']: config['DB_URL'] = 'sqlite+aiosqlite:///:memory:' - # config['DB_URL'] = 'sqlite+aiosqlite:///test.db' if config['DB_URL'] is None or config['DB_URL'] == '': raise Exception( @@ -92,6 +91,3 @@ async def deploy_schema(): async def close_engine(): 'Currently only needed when testing to force fixtures to refresh' await engine.dispose() - # Delete the data here for stateless testingĀ if not in-memory - # if config['TEST']: - # os.remove('test.db') diff --git a/server/npg/porchdb/data_access.py b/src/npg_porch/db/data_access.py similarity index 98% rename from server/npg/porchdb/data_access.py rename to src/npg_porch/db/data_access.py index effd0e4..67ad9c3 100644 --- a/server/npg/porchdb/data_access.py +++ b/src/npg_porch/db/data_access.py @@ -24,8 +24,8 @@ from sqlalchemy.orm import contains_eager, joinedload from sqlalchemy.orm.exc import NoResultFound -from npg.porchdb.models import Pipeline as DbPipeline, Task as DbTask, Event -from npg.porch.models import Task, Pipeline, TaskStateEnum +from npg_porch.db.models import Pipeline as DbPipeline, Task as DbTask, Event +from npg_porch.models import Task, Pipeline, TaskStateEnum class AsyncDbAccessor: diff --git a/server/npg/porchdb/models/__init__.py b/src/npg_porch/db/models/__init__.py similarity index 100% rename from server/npg/porchdb/models/__init__.py rename to src/npg_porch/db/models/__init__.py diff --git a/server/npg/porchdb/models/base.py b/src/npg_porch/db/models/base.py similarity index 100% rename from server/npg/porchdb/models/base.py rename to src/npg_porch/db/models/base.py diff --git a/server/npg/porchdb/models/event.py b/src/npg_porch/db/models/event.py similarity index 100% rename from server/npg/porchdb/models/event.py rename to src/npg_porch/db/models/event.py diff --git a/server/npg/porchdb/models/pipeline.py b/src/npg_porch/db/models/pipeline.py similarity index 96% rename from server/npg/porchdb/models/pipeline.py rename to src/npg_porch/db/models/pipeline.py index 1f03d5d..b4803a6 100644 --- a/server/npg/porchdb/models/pipeline.py +++ b/src/npg_porch/db/models/pipeline.py @@ -25,7 +25,7 @@ from .base import Base -from npg.porch.models import Pipeline as ModeledPipeline +from npg_porch.models import Pipeline as ModeledPipeline class Pipeline(Base): ''' diff --git a/server/npg/porchdb/models/task.py b/src/npg_porch/db/models/task.py similarity index 98% rename from server/npg/porchdb/models/task.py rename to src/npg_porch/db/models/task.py index dd146eb..91a7550 100644 --- a/server/npg/porchdb/models/task.py +++ b/src/npg_porch/db/models/task.py @@ -26,7 +26,7 @@ from sqlalchemy.sql.sqltypes import DateTime from .base import Base -from npg.porch.models import Task as ModelledTask +from npg_porch.models import Task as ModelledTask class Task(Base): diff --git a/server/npg/porchdb/models/token.py b/src/npg_porch/db/models/token.py similarity index 100% rename from server/npg/porchdb/models/token.py rename to src/npg_porch/db/models/token.py diff --git a/server/npg/porch/auth/__init__.py b/src/npg_porch/endpoints/__init__.py similarity index 100% rename from server/npg/porch/auth/__init__.py rename to src/npg_porch/endpoints/__init__.py diff --git a/server/npg/porch/endpoints/pipelines.py b/src/npg_porch/endpoints/pipelines.py similarity index 95% rename from server/npg/porch/endpoints/pipelines.py rename to src/npg_porch/endpoints/pipelines.py index d1afcd7..9a99604 100644 --- a/server/npg/porch/endpoints/pipelines.py +++ b/src/npg_porch/endpoints/pipelines.py @@ -25,10 +25,10 @@ from sqlalchemy.orm.exc import NoResultFound from starlette import status -from npg.porch.models.pipeline import Pipeline -from npg.porch.models.permission import RolesEnum -from npg.porchdb.connection import get_DbAccessor -from npg.porch.auth.token import validate +from npg_porch.models.pipeline import Pipeline +from npg_porch.models.permission import RolesEnum +from npg_porch.db.connection import get_DbAccessor +from npg_porch.auth.token import validate router = APIRouter( diff --git a/server/npg/porch/endpoints/tasks.py b/src/npg_porch/endpoints/tasks.py similarity index 95% rename from server/npg/porch/endpoints/tasks.py rename to src/npg_porch/endpoints/tasks.py index 3a57d2a..b4aecd1 100644 --- a/server/npg/porch/endpoints/tasks.py +++ b/src/npg_porch/endpoints/tasks.py @@ -22,11 +22,11 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException, Query -from npg.porch.auth.token import validate -from npg.porch.models.permission import PermissionValidationException -from npg.porch.models.pipeline import Pipeline -from npg.porch.models.task import Task, TaskStateEnum -from npg.porchdb.connection import get_DbAccessor +from npg_porch.auth.token import validate +from npg_porch.models.permission import PermissionValidationException +from npg_porch.models.pipeline import Pipeline +from npg_porch.models.task import Task, TaskStateEnum +from npg_porch.db.connection import get_DbAccessor from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound from starlette import status diff --git a/server/npg/porch/models/__init__.py b/src/npg_porch/models/__init__.py similarity index 100% rename from server/npg/porch/models/__init__.py rename to src/npg_porch/models/__init__.py diff --git a/server/npg/porch/models/permission.py b/src/npg_porch/models/permission.py similarity index 98% rename from server/npg/porch/models/permission.py rename to src/npg_porch/models/permission.py index df5b62c..783235f 100644 --- a/server/npg/porch/models/permission.py +++ b/src/npg_porch/models/permission.py @@ -22,7 +22,7 @@ from pydantic import BaseModel, Field, field_validator, FieldValidationInfo from typing import Optional -from npg.porch.models.pipeline import Pipeline +from npg_porch.models.pipeline import Pipeline class PermissionValidationException(Exception): diff --git a/server/npg/porch/models/pipeline.py b/src/npg_porch/models/pipeline.py similarity index 100% rename from server/npg/porch/models/pipeline.py rename to src/npg_porch/models/pipeline.py diff --git a/server/npg/porch/models/task.py b/src/npg_porch/models/task.py similarity index 98% rename from server/npg/porch/models/task.py rename to src/npg_porch/models/task.py index 788518d..df3f01e 100644 --- a/server/npg/porch/models/task.py +++ b/src/npg_porch/models/task.py @@ -23,7 +23,7 @@ import ujson from pydantic import BaseModel, Field -from npg.porch.models.pipeline import Pipeline +from npg_porch.models.pipeline import Pipeline class TaskStateEnum(str, Enum): PENDING = 'PENDING' diff --git a/server/npg/main.py b/src/npg_porch/server.py similarity index 97% rename from server/npg/main.py rename to src/npg_porch/server.py index 836928b..c7a9f8a 100644 --- a/server/npg/main.py +++ b/src/npg_porch/server.py @@ -21,7 +21,7 @@ from fastapi import FastAPI from fastapi.responses import HTMLResponse -from npg.porch.endpoints import pipelines, tasks +from npg_porch.endpoints import pipelines, tasks #https://fastapi.tiangolo.com/tutorial/bigger-applications/ #https://fastapi.tiangolo.com/tutorial/metadata diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py index 0a4ef75..337f633 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ -from .fixtures.orm_session import sync_session, async_session -from .fixtures.deploy_db import ( +from fixtures.orm_session import sync_session, async_session +from fixtures.deploy_db import ( sync_minimum, async_minimum, minimum_data, diff --git a/tests/data_access_test.py b/tests/data_access_test.py index c463b0c..bf1bccc 100644 --- a/tests/data_access_test.py +++ b/tests/data_access_test.py @@ -4,8 +4,8 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound -from npg.porchdb.data_access import AsyncDbAccessor -from npg.porch.models import Pipeline as ModelledPipeline, Task, TaskStateEnum +from npg_porch.db.data_access import AsyncDbAccessor +from npg_porch.models import Pipeline as ModelledPipeline, Task, TaskStateEnum def give_me_a_pipeline(number: int = 1): diff --git a/tests/db_auth_test.py b/tests/db_auth_test.py index e155452..625d493 100644 --- a/tests/db_auth_test.py +++ b/tests/db_auth_test.py @@ -2,16 +2,16 @@ import datetime from sqlalchemy import select -from npg.porchdb.models import Token, Pipeline -from npg.porchdb.auth import Validator, CredentialsValidationException -import npg.porch.models.permission -import npg.porch.models.pipeline +from npg_porch.db.models import Token, Pipeline +from npg_porch.db.auth import Validator, CredentialsValidationException +import npg_porch.models.permission +import npg_porch.models.pipeline @pytest.mark.asyncio async def test_token_string_is_valid(async_minimum): v = Validator(session = async_minimum) - assert isinstance(v, (npg.porchdb.auth.Validator)) + assert isinstance(v, (npg_porch.db.auth.Validator)) with pytest.raises(CredentialsValidationException, match=r'The token should be 32 chars long'): @@ -76,15 +76,15 @@ async def test_permission_object_is_returned(async_minimum): for t in token_rows: if t.description == 'Seqfarm host, job runner': p = await v.token2permission(t.token) - assert isinstance(p, (npg.porch.models.permission.Permission)) + assert isinstance(p, (npg_porch.models.permission.Permission)) assert p.pipeline is not None - assert isinstance(p.pipeline, (npg.porch.models.pipeline.Pipeline)) + assert isinstance(p.pipeline, (npg_porch.models.pipeline.Pipeline)) assert p.pipeline.name == 'ptest one' assert p.requestor_id == t.token_id assert p.role == 'regular_user' elif t.description == 'Seqfarm host, admin': p = await v.token2permission(t.token) - assert isinstance(p, (npg.porch.models.permission.Permission)) + assert isinstance(p, (npg_porch.models.permission.Permission)) assert p.pipeline is None assert p.requestor_id == t.token_id assert p.role == 'power_user' diff --git a/tests/db_task_test.py b/tests/db_task_test.py index 6672ea8..e1c9ec5 100644 --- a/tests/db_task_test.py +++ b/tests/db_task_test.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import select -from npg.porchdb.models import Task +from npg_porch.db.models import Task @pytest.mark.asyncio async def test_task_creation(async_minimum): diff --git a/tests/db_token_test.py b/tests/db_token_test.py index a7edbc0..0197b04 100644 --- a/tests/db_token_test.py +++ b/tests/db_token_test.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import select -from npg.porchdb.models import Token +from npg_porch.db.models import Token @pytest.mark.asyncio async def test_token_creation(async_minimum): diff --git a/tests/fixtures/deploy_db.py b/tests/fixtures/deploy_db.py index 1a0d966..ffecb99 100644 --- a/tests/fixtures/deploy_db.py +++ b/tests/fixtures/deploy_db.py @@ -3,12 +3,12 @@ import pytest_asyncio from starlette.testclient import TestClient -from npg.porchdb.models import ( +from npg_porch.db.models import ( Pipeline, Task, Event, Token ) -from npg.porchdb.data_access import AsyncDbAccessor -from npg.porch.models import Task as ModelledTask, TaskStateEnum -from npg.main import app +from npg_porch.db.data_access import AsyncDbAccessor +from npg_porch.models import Task as ModelledTask, TaskStateEnum +from npg_porch.server import app @pytest.fixture def minimum_data(): diff --git a/tests/fixtures/orm_session.py b/tests/fixtures/orm_session.py index 7cdbb09..3d5215f 100644 --- a/tests/fixtures/orm_session.py +++ b/tests/fixtures/orm_session.py @@ -4,8 +4,8 @@ import sqlalchemy import sqlalchemy.orm -from npg.porchdb.models import Base -from npg.porchdb.connection import session_factory, deploy_schema, close_engine +from npg_porch.db.models import Base +from npg_porch.db.connection import session_factory, deploy_schema, close_engine @pytest.fixture diff --git a/tests/init_test.py b/tests/init_test.py index 0f67135..9853441 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import select -from npg.porchdb.models import Pipeline +from npg_porch.db.models import Pipeline def test_fixture(sync_minimum): diff --git a/tests/model_permission_test.py b/tests/model_permission_test.py index 23d3716..80c42b4 100644 --- a/tests/model_permission_test.py +++ b/tests/model_permission_test.py @@ -1,7 +1,7 @@ import pytest -from npg.porch.models.pipeline import Pipeline -from npg.porch.models.permission import Permission, PermissionValidationException +from npg_porch.models.pipeline import Pipeline +from npg_porch.models.permission import Permission, PermissionValidationException from pydantic import ValidationError diff --git a/tests/pipeline_route_test.py b/tests/pipeline_route_test.py index bf0f5d8..fc455db 100644 --- a/tests/pipeline_route_test.py +++ b/tests/pipeline_route_test.py @@ -1,6 +1,6 @@ from starlette import status -from npg.porch.models import Pipeline +from npg_porch.models import Pipeline headers = { diff --git a/pytest.ini b/tests/pytest.ini similarity index 100% rename from pytest.ini rename to tests/pytest.ini diff --git a/tests/task_route_test.py b/tests/task_route_test.py index c24bd6a..043b40d 100644 --- a/tests/task_route_test.py +++ b/tests/task_route_test.py @@ -1,6 +1,6 @@ from starlette import status -from npg.porch.models import Task, TaskStateEnum, Pipeline +from npg_porch.models import Task, TaskStateEnum, Pipeline # Not testing get-all-tasks as this method will ultimately go