Merge pull request #57 from mgcam/update_top_level_name

Renamed the top-level source code directory.

README.md

@@ -36,11 +36,11 @@ To run the server, please execute the following from the root directory:
 ```bash
 bash
 pip3 install -e .
-cd server
+cd src
 mkdir -p logs
 export DB_URL=postgresql+asyncpg://npg_rw:$PASS@npg_porch_db:$PORT/$DATABASE
 export DB_SCHEMA='non_default'
-uvicorn npg.main:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json
+uvicorn npg_porch.server:app --host 0.0.0.0 --port 8080 --reload --log-config logging.json
 ```
 
 and open your browser at `http://localhost:8080` to see links to the docs.
@@ -52,7 +52,7 @@ The server will not start without `DB_URL` in the environment
 When you want HTTPS, logging and all that jazz:
 
 ```bash
-uvicorn main:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt
+uvicorn server:app --workers 2 --host 0.0.0.0 --port 8080 --log-config ~/logging.json --ssl-keyfile ~/.ssh/key.pem --ssl-certfile ~/.ssh/cert.pem --ssl-ca-certs /usr/local/share/ca-certificates/institute_ca.crt
 ```
 
 Consider running with nohup or similar.
@@ -62,7 +62,7 @@ Some notes on arguments:
 
 --host: 0.0.0.0 = bind to all network interfaces. Reliable but greedy in some situations
 
---log-config: Refers to a JSON file for python logging library. An example file is found in /server/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs.
+--log-config: Refers to a JSON file for python logging library. An example file is found in /src/logging.json. Uvicorn provides its own logging configuration via `uvicorn.access` and `uvicorn.error`. These may behave undesirably, and can be overridden in the JSON file with an alternate config. Likewise, fastapi logs to `fastapi` if that needs filtering. For logging to files, set `use_colors = False` in the relevant handlers or shell colour settings will appear as garbage in the logs.
 
 --ssl-keyfile: A PEM format key for the server certificate
 --ssl-certfile: A PEM format certificate for signing HTTPS communications
@@ -77,11 +77,11 @@ pip install -e .[test]
 pytest
 ```
 
-Individual tests are run in the form `pytest server/tests/init_test.py`
+Individual tests are run in the form `pytest tests/init_test.py`
 
 ### Fixtures
 
-Fixtures reside under `server/tests/fixtures` and are registered in `server/tests/conftest.py`
+Fixtures reside under `tests/fixtures` and are registered in `tests/conftest.py`
 They can also be listed by invoking `pytest --fixtures`
 
 Any fixtures that are not imported in `conftest.py` will not be detected.
@@ -106,7 +106,7 @@ The SET command ensures that the new schema is visible _for one session only_ in
 DB=npg_porch
 export DB_URL=postgresql+psycopg2://npg_admin:$PASS@npg_porch_db:$PORT/$DB
 # note that the script requires a regular PG driver, not the async version showed above
-server/deploy_schema.py
+src/deploy_schema.py
 
 psql --host=npg_porch_db --port=$PORT --username=npg_admin --password -d $DB
 ```

docs/user_guide.md

@@ -30,7 +30,7 @@ Access to the service is loosely controlled with authorisation tokens. You will
 
 ### Step 1 - register your pipeline with npg_porch
 
-*Schema: npg.porch.model.pipeline*
+*Schema: npg_porch.model.pipeline*
 
 Nothing in npg_porch can happen until there's a pipeline defined. For our purposes "pipeline" means "a thing you can run", and it may refer to specific code, or a wrapper that can run the pipeline in this particular way with some standard arguments.
 
@@ -50,15 +50,15 @@ You can name your pipeline however you like, but the name must be unique, and be
 
 Keep this pipeline definition with your data, as you will need it to tell npg_porch which pipeline you are acting on.
 
-When communicating with npg_porch (as with any HTTP server) you must inspect the response code and message after each communication. See `-w " %{http_code}" above. The API documentation lists the response codes you can expect to have to handle. In this case, the server may respond with 400 - BAD REQUEST if you leave out a name, or 409 - CONFLICT if you chose a name that is already created.
+As with any HTTP server, when communicating with npg_porch you must inspect the response code and message after each communication. See `-w " %{http_code}" above. The API documentation lists the response codes you can expect to have to handle. In this case, the server may respond with 400 - BAD REQUEST if you leave out a name, or 409 - CONFLICT if you chose a name that is already created.
 
 ### Step 2 - decide on the unique criteria for running the pipeline
 
 e.g. Once per 24 hours, poll iRODS metadata for data relating to a study.
 
 We might create a cronjob that runs a script. It invokes `imeta` and retrieves a list of results. Now we turn each of those results into a JSON document to our own specification:
 
-*Schema: npg.porch.model.task*
+*Schema: npg_porch.model.task*
 
 **study-100-id-run-45925.json**
 
@@ -113,7 +113,7 @@ Note that it is possible to run the same `task_input` with a different `pipeline
 
 ### Step 3 - register the documents with npg_porch
 
-*Schema: npg.porch.model.task*
+*Schema: npg_porch.model.task*
 
 Now you want the pipeline to run once per specification, and so register the documents with npg_porch.
 

scripts/deploy_schema.py

@@ -3,7 +3,7 @@
 import os
 import sqlalchemy
 
-import npg.porchdb.models
+import npg_porch.db.models
 
 db_url = os.environ.get('DB_URL')
 schema_name = os.environ.get('DB_SCHEMA')
@@ -17,5 +17,5 @@
     connect_args={'options': f'-csearch_path={schema_name}'}
 )
 
-npg.porchdb.models.Base.metadata.schema = schema_name
-npg.porchdb.models.Base.metadata.create_all(engine)
+npg_porch.db.models.Base.metadata.schema = schema_name
+npg_porch.db.models.Base.metadata.create_all(engine)

scripts/issue_token.py

@@ -5,7 +5,7 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.exc import NoResultFound
 
-from npg.porchdb.models import Token, Pipeline
+from npg_porch.db.models import Token, Pipeline
 
 parser = argparse.ArgumentParser(
     description='Creates a token in the backend DB and returns it'

server/npg/porch/auth/token.py → src/npg_porch/auth/token.py

@@ -23,8 +23,8 @@
 from fastapi.security import HTTPBearer
 from fastapi import HTTPException
 
-from npg.porchdb.connection import get_CredentialsValidator
-from npg.porchdb.auth import CredentialsValidationException
+from npg_porch.db.connection import get_CredentialsValidator
+from npg_porch.db.auth import CredentialsValidationException
 
 auth_scheme = HTTPBearer()
 

server/npg/porchdb/auth.py → src/npg_porch/db/auth.py

@@ -23,8 +23,8 @@
 from sqlalchemy.orm import contains_eager
 from sqlalchemy.orm.exc import NoResultFound
 
-from npg.porchdb.models import Token
-from npg.porch.models.permission import Permission, RolesEnum
+from npg_porch.db.models import Token
+from npg_porch.models.permission import Permission, RolesEnum
 
 __AUTH_TOKEN_LENGTH__ = 32
 __AUTH_TOKEN_REGEXP__ = re.compile(

server/npg/porchdb/connection.py → src/npg_porch/db/connection.py

@@ -22,9 +22,9 @@
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 
-from npg.porchdb.models import Base
-from npg.porchdb.data_access import AsyncDbAccessor
-from npg.porchdb.auth import Validator
+from npg_porch.db.models import Base
+from npg_porch.db.data_access import AsyncDbAccessor
+from npg_porch.db.auth import Validator
 
 config = {
     'DB_URL': os.environ.get('DB_URL'),
@@ -34,7 +34,6 @@
 
 if config['TEST']:
     config['DB_URL'] = 'sqlite+aiosqlite:///:memory:'
-    # config['DB_URL'] = 'sqlite+aiosqlite:///test.db'
 
 if config['DB_URL'] is None or config['DB_URL'] == '':
     raise Exception(
@@ -92,6 +91,3 @@ async def deploy_schema():
 async def close_engine():
     'Currently only needed when testing to force fixtures to refresh'
     await engine.dispose()
-    # Delete the data here for stateless testing if not in-memory
-    # if config['TEST']:
-    #     os.remove('test.db')

server/npg/porchdb/data_access.py → src/npg_porch/db/data_access.py

@@ -24,8 +24,8 @@
 from sqlalchemy.orm import contains_eager, joinedload
 from sqlalchemy.orm.exc import NoResultFound
 
-from npg.porchdb.models import Pipeline as DbPipeline, Task as DbTask, Event
-from npg.porch.models import Task, Pipeline, TaskStateEnum
+from npg_porch.db.models import Pipeline as DbPipeline, Task as DbTask, Event
+from npg_porch.models import Task, Pipeline, TaskStateEnum
 
 
 class AsyncDbAccessor:

server/npg/porchdb/models/pipeline.py → src/npg_porch/db/models/pipeline.py

@@ -25,7 +25,7 @@
 
 from .base import Base
 
-from npg.porch.models import Pipeline as ModeledPipeline
+from npg_porch.models import Pipeline as ModeledPipeline
 
 class Pipeline(Base):
     '''

server/npg/porchdb/models/task.py → src/npg_porch/db/models/task.py

@@ -26,7 +26,7 @@
 from sqlalchemy.sql.sqltypes import DateTime
 
 from .base import Base
-from npg.porch.models import Task as ModelledTask
+from npg_porch.models import Task as ModelledTask
 
 
 class Task(Base):

server/npg/porch/endpoints/pipelines.py → src/npg_porch/endpoints/pipelines.py

@@ -25,10 +25,10 @@
 from sqlalchemy.orm.exc import NoResultFound
 from starlette import status
 
-from npg.porch.models.pipeline import Pipeline
-from npg.porch.models.permission import RolesEnum
-from npg.porchdb.connection import get_DbAccessor
-from npg.porch.auth.token import validate
+from npg_porch.models.pipeline import Pipeline
+from npg_porch.models.permission import RolesEnum
+from npg_porch.db.connection import get_DbAccessor
+from npg_porch.auth.token import validate
 
 
 router = APIRouter(

server/npg/porch/endpoints/tasks.py → src/npg_porch/endpoints/tasks.py

@@ -22,11 +22,11 @@
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, HTTPException, Query
-from npg.porch.auth.token import validate
-from npg.porch.models.permission import PermissionValidationException
-from npg.porch.models.pipeline import Pipeline
-from npg.porch.models.task import Task, TaskStateEnum
-from npg.porchdb.connection import get_DbAccessor
+from npg_porch.auth.token import validate
+from npg_porch.models.permission import PermissionValidationException
+from npg_porch.models.pipeline import Pipeline
+from npg_porch.models.task import Task, TaskStateEnum
+from npg_porch.db.connection import get_DbAccessor
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 from starlette import status

server/npg/porch/models/permission.py → src/npg_porch/models/permission.py

@@ -22,7 +22,7 @@
 from pydantic import BaseModel, Field, field_validator, FieldValidationInfo
 from typing import Optional
 
-from npg.porch.models.pipeline import Pipeline
+from npg_porch.models.pipeline import Pipeline
 
 
 class PermissionValidationException(Exception):

server/npg/porch/models/task.py → src/npg_porch/models/task.py

@@ -23,7 +23,7 @@
 import ujson
 from pydantic import BaseModel, Field
 
-from npg.porch.models.pipeline import Pipeline
+from npg_porch.models.pipeline import Pipeline
 
 class TaskStateEnum(str, Enum):
     PENDING = 'PENDING'

server/npg/main.py → src/npg_porch/server.py

@@ -21,7 +21,7 @@
 from fastapi import FastAPI
 from fastapi.responses import HTMLResponse
 
-from npg.porch.endpoints import pipelines, tasks
+from npg_porch.endpoints import pipelines, tasks
 
 #https://fastapi.tiangolo.com/tutorial/bigger-applications/
 #https://fastapi.tiangolo.com/tutorial/metadata

tests/conftest.py

@@ -1,5 +1,5 @@
-from .fixtures.orm_session import sync_session, async_session
-from .fixtures.deploy_db import (
+from fixtures.orm_session import sync_session, async_session
+from fixtures.deploy_db import (
     sync_minimum,
     async_minimum,
     minimum_data,

tests/data_access_test.py

@@ -4,8 +4,8 @@
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 
-from npg.porchdb.data_access import AsyncDbAccessor
-from npg.porch.models import Pipeline as ModelledPipeline, Task, TaskStateEnum
+from npg_porch.db.data_access import AsyncDbAccessor
+from npg_porch.models import Pipeline as ModelledPipeline, Task, TaskStateEnum
 
 
 def give_me_a_pipeline(number: int = 1):

tests/db_auth_test.py

@@ -2,16 +2,16 @@
 import datetime
 from sqlalchemy import select
 
-from npg.porchdb.models import Token, Pipeline
-from npg.porchdb.auth import Validator, CredentialsValidationException
-import npg.porch.models.permission
-import npg.porch.models.pipeline
+from npg_porch.db.models import Token, Pipeline
+from npg_porch.db.auth import Validator, CredentialsValidationException
+import npg_porch.models.permission
+import npg_porch.models.pipeline
 
 @pytest.mark.asyncio
 async def test_token_string_is_valid(async_minimum):
 
     v = Validator(session = async_minimum)
-    assert isinstance(v, (npg.porchdb.auth.Validator))
+    assert isinstance(v, (npg_porch.db.auth.Validator))
 
     with pytest.raises(CredentialsValidationException,
                        match=r'The token should be 32 chars long'):
@@ -76,15 +76,15 @@ async def test_permission_object_is_returned(async_minimum):
     for t in token_rows:
         if t.description == 'Seqfarm host, job runner':
             p = await v.token2permission(t.token)
-            assert isinstance(p, (npg.porch.models.permission.Permission))
+            assert isinstance(p, (npg_porch.models.permission.Permission))
             assert p.pipeline is not None
-            assert isinstance(p.pipeline, (npg.porch.models.pipeline.Pipeline))
+            assert isinstance(p.pipeline, (npg_porch.models.pipeline.Pipeline))
             assert p.pipeline.name == 'ptest one'
             assert p.requestor_id == t.token_id
             assert p.role == 'regular_user'
         elif t.description == 'Seqfarm host, admin':
             p = await v.token2permission(t.token)
-            assert isinstance(p, (npg.porch.models.permission.Permission))
+            assert isinstance(p, (npg_porch.models.permission.Permission))
             assert p.pipeline is None
             assert p.requestor_id == t.token_id
             assert p.role == 'power_user'

tests/db_task_test.py

@@ -1,7 +1,7 @@
 import pytest
 from sqlalchemy import select
 
-from npg.porchdb.models import Task
+from npg_porch.db.models import Task
 
 @pytest.mark.asyncio
 async def test_task_creation(async_minimum):

tests/db_token_test.py

@@ -1,7 +1,7 @@
 import pytest
 from sqlalchemy import select
 
-from npg.porchdb.models import Token
+from npg_porch.db.models import Token
 
 @pytest.mark.asyncio
 async def test_token_creation(async_minimum):

tests/fixtures/deploy_db.py

@@ -3,12 +3,12 @@
 import pytest_asyncio
 from starlette.testclient import TestClient
 
-from npg.porchdb.models import (
+from npg_porch.db.models import (
     Pipeline, Task, Event, Token
 )
-from npg.porchdb.data_access import AsyncDbAccessor
-from npg.porch.models import Task as ModelledTask, TaskStateEnum
-from npg.main import app
+from npg_porch.db.data_access import AsyncDbAccessor
+from npg_porch.models import Task as ModelledTask, TaskStateEnum
+from npg_porch.server import app
 
 @pytest.fixture
 def minimum_data():

tests/fixtures/orm_session.py

@@ -4,8 +4,8 @@
 import sqlalchemy
 import sqlalchemy.orm
 
-from npg.porchdb.models import Base
-from npg.porchdb.connection import session_factory, deploy_schema, close_engine
+from npg_porch.db.models import Base
+from npg_porch.db.connection import session_factory, deploy_schema, close_engine
 
 
 @pytest.fixture

tests/init_test.py

@@ -1,7 +1,7 @@
 import pytest
 from sqlalchemy import select
 
-from npg.porchdb.models import Pipeline
+from npg_porch.db.models import Pipeline
 
 
 def test_fixture(sync_minimum):

tests/model_permission_test.py

@@ -1,7 +1,7 @@
 import pytest
 
-from npg.porch.models.pipeline import Pipeline
-from npg.porch.models.permission import Permission, PermissionValidationException
+from npg_porch.models.pipeline import Pipeline
+from npg_porch.models.permission import Permission, PermissionValidationException
 from pydantic import ValidationError