Merge pull request #37 from punch-mission/develop

Makes pre-commit pass

.gitignore

@@ -4,4 +4,4 @@ scripts/credentials.py
 /venv-docs/
 build/
 punchpipe.egg-info
-.idea
+.idea

README.md

@@ -1,26 +1,26 @@
 # punchpipe
-`punchpipe` is the data processing pipeline for [the PUNCH mission](https://punch.space.swri.edu/). 
+`punchpipe` is the data processing pipeline for [the PUNCH mission](https://punch.space.swri.edu/).
 All the science code and actual calibration functionality lives in `punchbowl`. This package
-only automates the control segment for the Science Operations Center. 
+only automates the control segment for the Science Operations Center.
 
 The `punchpipe` is organized into segments, i.e. levels of processing to produce specific
-data products. Segments are referred in code by their ending level, 
-e.g. `level1` means the Level 0 to Level 1 segment. 
+data products. Segments are referred in code by their ending level,
+e.g. `level1` means the Level 0 to Level 1 segment.
 
 ## Accessing the data
 
 ## First-time setup
-1. Create a clean virtual environment. You can do this with conda using `conda env create --name ENVIRONMENT-NAME` 
-2. Install `punchbowl` using `pip install .` in the `punchbowl` directory. 
-3. Install `punchpipe` using `pip install .` while in this directory 
-4. Set up database credentials Prefect block by running `python scripts/credentials.py`. 
-    - If this file does not exist for you. You need to determine your mySQL credentials then create a block in Python: 
+1. Create a clean virtual environment. You can do this with conda using `conda env create --name ENVIRONMENT-NAME`
+2. Install `punchbowl` using `pip install .` in the `punchbowl` directory.
+3. Install `punchpipe` using `pip install .` while in this directory
+4. Set up database credentials Prefect block by running `python scripts/credentials.py`.
+    - If this file does not exist for you. You need to determine your mySQL credentials then create a block in Python:
     ```py
    from punchpipe.controlsegment.db import MySQLCredentials
    cred = MySQLCredentials(user="username", password="password")
    cred.save('mysql-cred')
     ```
-5. Set up databases by running `scripts/create_db.py` directory. 
+5. Set up databases by running `scripts/create_db.py` directory.
 6. Build all the necessary deployments for Prefect by following [these instructions](https://docs.prefect.io/concepts/deployments/).
    - See below for an example:
    ```shell
@@ -35,9 +35,9 @@ e.g. `level1` means the Level 0 to Level 1 segment.
 3. Create agents for the work queues by following the instructions in the UI for the work queue
 
 ## Resetting
-1. Reset the Prefect Orion database using `prefect orion database reset`. 
+1. Reset the Prefect Orion database using `prefect orion database reset`.
 2. Remove all the `punchpipe` databases by running `erase_db.sql`
 
 ## Contributing
- 
-## Licensing
+
+## Licensing

docs/source/control/configuration.rst

@@ -1,4 +1,4 @@
 Configuration
 ==============
 
-How to set up a configuration file.
+How to set up a configuration file.

docs/source/control/index.rst

@@ -6,4 +6,4 @@ Control Segment Design
    :caption: Contents:
 
    states
-   configuration
+   configuration

old_config.yaml

@@ -25,4 +25,4 @@ scheduler:
 
 process_options:
   pointing:
-    num_quads: 100
+    num_quads: 100

punchpipe/controlsegment/__init__.py

@@ -1,13 +1,12 @@
 from datetime import datetime
 
-import pytest
 import numpy as np
+import pytest
 from astropy.nddata import StdDevUncertainty
 from astropy.wcs import WCS
 from punchbowl.data import NormalizedMetadata, PUNCHData
 
-from punchpipe.controlsegment.db import Base, Flow, File
-from punchpipe.controlsegment.processor import generic_process_flow_logic
+from punchpipe.controlsegment.db import File
 from punchpipe.controlsegment.util import match_data_with_file_db_entry
 
 

punchpipe/controlsegment/db.py

@@ -1,10 +1,10 @@
-from typing import Optional
 import os
+from typing import Optional
 
 from prefect.blocks.core import Block
 from pydantic import SecretStr
-from sqlalchemy import Column, ForeignKey, Integer, String, create_engine, DateTime, Float, TEXT
-from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy import TEXT, Column, DateTime, Integer, String
+from sqlalchemy.orm import declarative_base
 
 
 class MySQLCredentials(Block):
@@ -84,4 +84,3 @@ class FileRelationship(Base):
     relationship_id = Column(Integer, primary_key=True)
     parent = Column(Integer, nullable=False)
     child = Column(Integer, nullable=False)
-

punchpipe/controlsegment/launcher.py

@@ -1,13 +1,14 @@
-from typing import List
 from datetime import datetime, timedelta
+from typing import List
 
-from prefect import task, flow, get_run_logger
+from prefect import flow, get_run_logger, task
 from prefect.client import get_client
 from sqlalchemy import and_
 from sqlalchemy.orm import Session
 
 from punchpipe.controlsegment.db import Flow
-from punchpipe.controlsegment.util import get_database_session, load_pipeline_configuration
+from punchpipe.controlsegment.util import (get_database_session,
+                                           load_pipeline_configuration)
 
 
 @task

punchpipe/controlsegment/processor.py

@@ -1,14 +1,13 @@
-from datetime import datetime
 import json
-import os
+from datetime import datetime
 
 from prefect.context import get_run_context
 
-from punchpipe.controlsegment.db import Flow, File
+from punchpipe.controlsegment.db import File, Flow
 from punchpipe.controlsegment.util import (get_database_session,
                                            load_pipeline_configuration,
-                                           write_file,
-                                           match_data_with_file_db_entry)
+                                           match_data_with_file_db_entry,
+                                           write_file)
 
 
 def generic_process_flow_logic(flow_id: int, core_flow_to_launch,

punchpipe/controlsegment/scheduler.py

@@ -1,7 +1,8 @@
-import itertools
 
 from punchpipe.controlsegment.db import File, FileRelationship
-from punchpipe.controlsegment.util import get_database_session, update_file_state, load_pipeline_configuration
+from punchpipe.controlsegment.util import (get_database_session,
+                                           load_pipeline_configuration,
+                                           update_file_state)
 
 
 def generic_scheduler_flow_logic(query_ready_files_func,

punchpipe/controlsegment/tests/conftest.py

@@ -4,4 +4,4 @@
 
 @pytest.fixture(scope='session')
 def pmr_mysql_config():
-    return MysqlConfig(image='mariadb:latest')
+    return MysqlConfig(image='mariadb:latest')

punchpipe/controlsegment/tests/test_launcher.py

@@ -1,17 +1,16 @@
-from datetime import datetime
 import os
+from datetime import datetime
 
-from pytest_mock_resources import create_mysql_fixture
-from prefect.testing.utilities import prefect_test_harness
-from prefect.logging import disable_run_logger
 from freezegun import freeze_time
+from prefect.logging import disable_run_logger
+from prefect.testing.utilities import prefect_test_harness
+from pytest_mock_resources import create_mysql_fixture
 
-from punchpipe import __version__
-from punchpipe.controlsegment.db import Base, Flow, File
-from punchpipe.controlsegment.launcher import (gather_planned_flows,
-                                               count_running_flows,
+from punchpipe.controlsegment.db import Base, File, Flow
+from punchpipe.controlsegment.launcher import (count_running_flows,
+                                               escalate_long_waiting_flows,
                                                filter_for_launchable_flows,
-                                               escalate_long_waiting_flows)
+                                               gather_planned_flows)
 from punchpipe.controlsegment.util import load_pipeline_configuration
 
 TEST_DIR = os.path.dirname(__file__)
@@ -132,7 +131,3 @@ def test_filter_for_launchable_flows_with_empty_db(db_empty):
 
 def test_launch_ready_flows():
     pass
-
-
-
-

punchpipe/controlsegment/tests/test_processor.py

@@ -1,20 +1,19 @@
-from datetime import datetime
 import json
 import os
 import shutil
+from datetime import datetime
 
-import pytest
-from prefect import flow, task
-from prefect.testing.utilities import prefect_test_harness
-from pytest_mock_resources import create_mysql_fixture
 import numpy as np
+import pytest
 from astropy.nddata import StdDevUncertainty
 from astropy.wcs import WCS
+from prefect import flow
+from prefect.testing.utilities import prefect_test_harness
 from punchbowl.data import NormalizedMetadata, PUNCHData
+from pytest_mock_resources import create_mysql_fixture
 
-from punchpipe.controlsegment.db import Base, Flow, File
+from punchpipe.controlsegment.db import Base, File, Flow
 from punchpipe.controlsegment.processor import generic_process_flow_logic
-from punchpipe.controlsegment.util import match_data_with_file_db_entry
 
 TESTDATA_DIR = os.path.dirname(__file__)
 

punchpipe/controlsegment/util.py

@@ -1,14 +1,13 @@
 import os
 
+import yaml
+from prefect import task
+from punchbowl.data import PUNCHData
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
-from prefect import task
-import yaml
 from yaml.loader import FullLoader
-from punchbowl.data import PUNCHData
 
-from punchpipe.controlsegment.db import MySQLCredentials
-from punchpipe.controlsegment.db import Flow, File, FileRelationship
+from punchpipe.controlsegment.db import File, MySQLCredentials
 
 
 def get_database_session():

punchpipe/flows/level1.py

@@ -1,15 +1,14 @@
-from datetime import datetime
 import json
 import os
 import typing as t
+from datetime import datetime
 
-from sqlalchemy import and_
 from prefect import flow, task
-from prefect.context import get_run_context
 from punchbowl.level1.flow import level1_core_flow
+from sqlalchemy import and_
 
 from punchpipe import __version__
-from punchpipe.controlsegment.db import Flow, File
+from punchpipe.controlsegment.db import File, Flow
 from punchpipe.controlsegment.processor import generic_process_flow_logic
 from punchpipe.controlsegment.scheduler import generic_scheduler_flow_logic
 

punchpipe/flows/level2.py

@@ -1,14 +1,14 @@
-from datetime import datetime, timedelta
 import json
 import os
 import typing as t
+from datetime import datetime, timedelta
 
-from sqlalchemy import and_
 from prefect import flow, task
 from punchbowl.level2.flow import level2_core_flow
+from sqlalchemy import and_
 
 from punchpipe import __version__
-from punchpipe.controlsegment.db import Flow, File
+from punchpipe.controlsegment.db import File, Flow
 from punchpipe.controlsegment.processor import generic_process_flow_logic
 from punchpipe.controlsegment.scheduler import generic_scheduler_flow_logic
 

punchpipe/flows/tests/conftest.py

@@ -4,4 +4,4 @@
 
 @pytest.fixture(scope='session')
 def pmr_mysql_config():
-    return MysqlConfig(image='mariadb:latest')
+    return MysqlConfig(image='mariadb:latest')

punchpipe/flows/tests/test_level1.py

@@ -1,13 +1,16 @@
-from datetime import datetime
 import os
+from datetime import datetime
 
-from pytest_mock_resources import create_mysql_fixture
 from prefect.testing.utilities import prefect_test_harness
+from pytest_mock_resources import create_mysql_fixture
 
 from punchpipe import __version__
-from punchpipe.controlsegment.db import Base, Flow, File
+from punchpipe.controlsegment.db import Base, File, Flow
 from punchpipe.controlsegment.util import load_pipeline_configuration
-from punchpipe.flows.level1 import level1_query_ready_files, level1_construct_file_info, level1_construct_flow_info, level1_scheduler_flow
+from punchpipe.flows.level1 import (level1_construct_file_info,
+                                    level1_construct_flow_info,
+                                    level1_query_ready_files,
+                                    level1_scheduler_flow)
 
 TEST_DIR = os.path.dirname(__file__)
 
@@ -84,7 +87,7 @@ def test_level1_construct_flow_info():
 def test_level1_scheduler_flow(db):
     pipeline_config_path = os.path.join(TEST_DIR, "config.yaml")
     with prefect_test_harness():
-        outcome = level1_scheduler_flow(pipeline_config_path, db)
+        level1_scheduler_flow(pipeline_config_path, db)
     results = db.query(Flow).where(Flow.state == 'planned').all()
     assert len(results) == 1
 

punchpipe/flows/tests/test_level2.py

@@ -1,14 +1,17 @@
-from datetime import datetime
 import os
+from datetime import datetime
 
-from pytest_mock_resources import create_mysql_fixture
-from prefect.testing.utilities import prefect_test_harness
 from freezegun import freeze_time
+from prefect.testing.utilities import prefect_test_harness
+from pytest_mock_resources import create_mysql_fixture
 
 from punchpipe import __version__
-from punchpipe.controlsegment.db import Base, Flow, File
+from punchpipe.controlsegment.db import Base, File, Flow
 from punchpipe.controlsegment.util import load_pipeline_configuration
-from punchpipe.flows.level2 import level2_query_ready_files, level2_construct_file_info, level2_construct_flow_info, level2_scheduler_flow
+from punchpipe.flows.level2 import (level2_construct_file_info,
+                                    level2_construct_flow_info,
+                                    level2_query_ready_files,
+                                    level2_scheduler_flow)
 
 TEST_DIR = os.path.dirname(__file__)
 
@@ -38,7 +41,7 @@ def session_fn(session):
 
 
 def test_level2_query_ready_files(db):
-    with freeze_time(datetime(2023, 1, 1, 0, 5, 0)) as frozen_datatime:
+    with freeze_time(datetime(2023, 1, 1, 0, 5, 0)) as frozen_datatime:  # noqa: F841
         pipeline_config = {'levels': {'level2_process_flow': {'schedule': {'latency': 3, 'window_duration_seconds': 3}}}}
         ready_file_ids = level2_query_ready_files.fn(db, pipeline_config)
         assert len(ready_file_ids) == 1
@@ -88,7 +91,7 @@ def test_level2_construct_flow_info():
 def test_level2_scheduler_flow(db):
     pipeline_config_path = os.path.join(TEST_DIR, "config.yaml")
     with prefect_test_harness():
-        outcome = level2_scheduler_flow(pipeline_config_path, db)
+        level2_scheduler_flow(pipeline_config_path, db)
     results = db.query(Flow).where(Flow.state == 'planned').all()
     assert len(results) == 0
 

punchpipe/monitor/app.py

@@ -1,13 +1,13 @@
 # Run this app with `python app.py` and
 # visit http://127.0.0.1:8050/ in your web browser.
 
-from dash import Dash, dcc, html, Input, Output, State, dash_table
-import plotly.express as px
+import json
+from datetime import date, timedelta
+
 import pandas as pd
-import numpy as np
-from datetime import timedelta, date
+import plotly.express as px
+from dash import Dash, Input, Output, dash_table, dcc, html
 from dash.exceptions import PreventUpdate
-import json
 
 app = Dash(__name__)
 

punchpipe/monitor/create_fake_data.py

@@ -1,10 +1,12 @@
 import random
-import numpy as np
+import uuid
+from datetime import datetime, timedelta
+from random import randrange
+
 import coolname
+import numpy as np
 import pandas as pd
-from random import randrange
-from datetime import datetime, timedelta
-import uuid
+
 
 def random_date(start, end):
     """