From 677d028a3a376477a51da72e7e49c013d6b0f7f5 Mon Sep 17 00:00:00 2001 From: jtmaze Date: Thu, 22 Aug 2024 14:36:29 -0700 Subject: [PATCH 01/16] added campaign information table class --- snowexsql/tables/campaign_data.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 snowexsql/tables/campaign_data.py diff --git a/snowexsql/tables/campaign_data.py b/snowexsql/tables/campaign_data.py new file mode 100644 index 0000000..b56988e --- /dev/null +++ b/snowexsql/tables/campaign_data.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Aug 22 11:56:34 2024 + +@author: jtmaz +""" + +from sqlalchemy import Column, String, Integer + +from .base import Base + + +class CampaignData(Base): + """ + Table stores Campaign data. Does not store data values, + it only stores the campaign metadata. + """ + __tablename__ = 'sites' + __table_args__ = {"schema": "public"} + + id = Column(Integer, primary_key=True) + name = Column(String(100)) + description = Column(String(1000)) + + \ No newline at end of file From 9b4f64d439f90cac87c26d7403c7edd9befc4d0e Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 22 Aug 2024 12:28:49 -0600 Subject: [PATCH 02/16] add observers and instruments tables, linking pointdata --- snowexsql/tables/__init__.py | 5 ++++- snowexsql/tables/base.py | 2 -- snowexsql/tables/instrument.py | 17 +++++++++++++++++ snowexsql/tables/observers.py | 15 +++++++++++++++ snowexsql/tables/point_data.py | 35 +++++++++++++++++++++++++++++++++- tests/sql_test_base.py | 19 +++++++++++++++--- tests/test_db.py | 4 ++-- 7 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 snowexsql/tables/instrument.py create mode 100644 snowexsql/tables/observers.py diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index 3a43124..c778d55 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -2,10 +2,13 @@ from .layer_data import LayerData from .point_data import PointData from .site_data import SiteData +from .observers import Observer +from .instrument import Instrument __all__ = [ 'ImageData', 'LayerData', 'PointData', - 'SnowData', + "Instrument", + "Observer", ] diff --git a/snowexsql/tables/base.py b/snowexsql/tables/base.py index eb6a915..e0772a4 100644 --- a/snowexsql/tables/base.py +++ b/snowexsql/tables/base.py @@ -45,7 +45,5 @@ class Measurement(object): """ Base Class providing attributes required for a measurement of any kind """ - instrument = Column(String(50)) type = Column(String(50)) units = Column(String(50)) - observers = Column(String(100)) diff --git a/snowexsql/tables/instrument.py b/snowexsql/tables/instrument.py new file mode 100644 index 0000000..9d40717 --- /dev/null +++ b/snowexsql/tables/instrument.py @@ -0,0 +1,17 @@ +from sqlalchemy import Column, Integer, String +from .base import Base + + +class Instrument(Base): + __tablename__ = 'instruments' + __table_args__ = {"schema": "public"} + # this seems important + __mapper_args__ = { + 'polymorphic_identity': 'Instruments', + } + # auto created id + id = Column(Integer, primary_key=True) + # Name of the instrument + name = Column(String(50)) + model = Column(String(255)) + specifications = Column(String(255)) diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py new file mode 100644 index 0000000..02f5515 --- /dev/null +++ b/snowexsql/tables/observers.py @@ -0,0 +1,15 @@ +from sqlalchemy.orm import mapped_column, Mapped +from sqlalchemy import Column, String + +from .base import Base + + +class Observer(Base): + __tablename__ = 'observers' + __table_args__ = {"schema": "public"} + # id is mapped column for many-to-many + id: Mapped[int] = mapped_column(primary_key=True) + # Name of the observer + first_name = Column(String(255)) + last_name = Column(String(255)) + email = Column(String(255)) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index c6189ee..30b4ff3 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -1,6 +1,24 @@ -from sqlalchemy import Column, Float, Integer, String +from sqlalchemy import Column, Float, Integer, String, ForeignKey +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column +from typing import List +from sqlalchemy.orm import relationship from .base import Base, Measurement, SingleLocationData +from .observers import Observer +from .instrument import Instrument + + +class PointObservers(Base): + """ + Link table + """ + __tablename__ = 'point_observers' + __table_args__ = {'schema': 'public'} + + id = Column(Integer, primary_key=True) + point_id = Column(Integer, ForeignKey('public.points.id')) + observer_id = Column(Integer, ForeignKey("public.observers.id")) class PointData(SingleLocationData, Measurement, Base): @@ -14,3 +32,18 @@ class PointData(SingleLocationData, Measurement, Base): version_number = Column(Integer) equipment = Column(String(50)) value = Column(Float) + + # bring these in instead of Measurement + type = Column(String(50)) + units = Column(String(50)) + + # This should be the foreign key + # instrument = Column(String(50)) + # Link the instrument id with a foreign key + instrument_id = Column(Integer, ForeignKey('public.instruments.id')) + # Link the Instrument class + instrument = relationship('Instrument') + + # id is a mapped column for many-to-many with observers + id: Mapped[int] = mapped_column(primary_key=True) + observers: Mapped[List[Observer]] = relationship(secondary=PointObservers) diff --git a/tests/sql_test_base.py b/tests/sql_test_base.py index 2ef6934..7fbeeb0 100644 --- a/tests/sql_test_base.py +++ b/tests/sql_test_base.py @@ -1,7 +1,7 @@ from os.path import dirname, join from numpy.testing import assert_almost_equal -from sqlalchemy import asc +from sqlalchemy import asc, text from snowexsql.db import get_db, initialize @@ -42,9 +42,22 @@ def setup_class(self): @classmethod def teardown_class(self): """ - Remove the databse + Remove the database """ - self.metadata.drop_all(bind=self.engine) + # self.metadata.tables['point_observers'].drop( + # self.engine, checkfirst=True + # ) + # self.metadata.tables['observers'].drop( + # self.engine, checkfirst=True + # ) + # self.metadata.tables['points'].drop( + # self.engine, checkfirst=True + # ) + # self.metadata.tables['instruments'].drop( + # self.engine, checkfirst=True + # ) + # self.metadata.drop_all(bind=self.engine, checkfirst=True) + # self.metadata.drop_all(bind=self.engine) self.session.close() # optional, depends on use case def teardown(self): diff --git a/tests/test_db.py b/tests/test_db.py index 18d0df4..35b979d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -12,7 +12,7 @@ class TestDB(DBSetup): base_atts = ['site_name', 'date', 'site_id'] single_loc_atts = ['elevation', 'geom', 'time'] - meas_atts = ['instrument', 'type', 'units', 'observers'] + meas_atts = ['type', 'units'] site_atts = base_atts + single_loc_atts + \ ['slope_angle', 'aspect', 'air_temp', 'total_depth', @@ -22,7 +22,7 @@ class TestDB(DBSetup): 'tree_canopy', 'site_notes'] point_atts = single_loc_atts + meas_atts + \ - ['version_number', 'equipment', 'value'] + ['version_number', 'equipment', 'value', 'instrument_id'] layer_atts = single_loc_atts + meas_atts + \ ['depth', 'value', 'bottom_depth', 'comments', 'sample_a', From 0e15b90b7b125b1967a10c1cb10a66b83efe2e9d Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 22 Aug 2024 12:43:31 -0600 Subject: [PATCH 03/16] We don't need to drop all after the tests, so get rid of that commented out code --- tests/sql_test_base.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/sql_test_base.py b/tests/sql_test_base.py index 7fbeeb0..5222952 100644 --- a/tests/sql_test_base.py +++ b/tests/sql_test_base.py @@ -42,22 +42,8 @@ def setup_class(self): @classmethod def teardown_class(self): """ - Remove the database + Close the session """ - # self.metadata.tables['point_observers'].drop( - # self.engine, checkfirst=True - # ) - # self.metadata.tables['observers'].drop( - # self.engine, checkfirst=True - # ) - # self.metadata.tables['points'].drop( - # self.engine, checkfirst=True - # ) - # self.metadata.tables['instruments'].drop( - # self.engine, checkfirst=True - # ) - # self.metadata.drop_all(bind=self.engine, checkfirst=True) - # self.metadata.drop_all(bind=self.engine) self.session.close() # optional, depends on use case def teardown(self): From 531562fdfe19cebb33fdbd76b5917c1b9131f525 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 22 Aug 2024 16:49:53 -0600 Subject: [PATCH 04/16] Tests are mostly passing, including with fake data. Querying still needs work in the API --- snowexsql/api.py | 26 +++++++--- snowexsql/tables/observers.py | 1 - snowexsql/tables/point_data.py | 6 +-- tests/test_api.py | 87 +++++++++++++++++++++++++++------- 4 files changed, 92 insertions(+), 28 deletions(-) diff --git a/snowexsql/api.py b/snowexsql/api.py index 616e51a..6652737 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -10,7 +10,9 @@ from snowexsql.conversions import query_to_geopandas, raster_to_rasterio from snowexsql.db import get_db -from snowexsql.tables import ImageData, LayerData, PointData +from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ + Observer +from snowexsql.tables.point_data import PointObservers LOG = logging.getLogger(__name__) DB_NAME = 'snow:hackweek@db.snowexdata.org/snowex' @@ -47,8 +49,10 @@ class BaseDataset: # Use this database name DB_NAME = DB_NAME - ALLOWED_QRY_KWARGS = ["site_name", "site_id", "date", "instrument", "observers", "type", - "utm_zone", "date_greater_equal", "date_less_equal", "value_greater_equal", 'value_less_equal', + ALLOWED_QRY_KWARGS = [ + "site_name", "site_id", "date", "instrument", "type", + "utm_zone", "date_greater_equal", "date_less_equal", + "value_greater_equal", 'value_less_equal', ] SPECIAL_KWARGS = ["limit"] # Default max record count @@ -119,6 +123,11 @@ def extend_qry(cls, qry, check_size=True, **kwargs): key = k.split("_less_equal")[0] filter_col = getattr(cls.MODEL, key) qry = qry.filter(filter_col <= v) + # Filter linked columns + elif k == "instrument": + qry = qry.filter( + cls.MODEL.instrument.has(name=v) + ) # Filter to exact value else: filter_col = getattr(cls.MODEL, k) @@ -198,9 +207,12 @@ def all_observers(self): Return all distinct observers in the data """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(self.MODEL.observers).distinct() + qry = session.query(Observer).join( + PointObservers, Observer.id == PointObservers.observer_id + ).distinct() result = qry.all() - return self.retrieve_single_value_result(result) + # Join the names + return [f"{r.first_name} {r.last_name}" for r in result] @property def all_units(self): @@ -218,7 +230,9 @@ def all_instruments(self): Return all distinct instruments in the data """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(self.MODEL.instrument).distinct() + qry = session.query(Instrument.name).join( + PointData, Instrument.id == PointData.instrument_id + ).distinct() result = qry.all() return self.retrieve_single_value_result(result) diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py index 02f5515..0cf59d0 100644 --- a/snowexsql/tables/observers.py +++ b/snowexsql/tables/observers.py @@ -12,4 +12,3 @@ class Observer(Base): # Name of the observer first_name = Column(String(255)) last_name = Column(String(255)) - email = Column(String(255)) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index 30b4ff3..e421d43 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -37,8 +37,6 @@ class PointData(SingleLocationData, Measurement, Base): type = Column(String(50)) units = Column(String(50)) - # This should be the foreign key - # instrument = Column(String(50)) # Link the instrument id with a foreign key instrument_id = Column(Integer, ForeignKey('public.instruments.id')) # Link the Instrument class @@ -46,4 +44,6 @@ class PointData(SingleLocationData, Measurement, Base): # id is a mapped column for many-to-many with observers id: Mapped[int] = mapped_column(primary_key=True) - observers: Mapped[List[Observer]] = relationship(secondary=PointObservers) + observers: Mapped[List[Observer]] = relationship( + secondary=PointObservers.__table__ + ) diff --git a/tests/test_api.py b/tests/test_api.py index 46fa128..adcb0c9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,12 +2,13 @@ import geopandas as gpd import numpy as np import pytest -from datetime import date - +from datetime import date, time +from geoalchemy2.elements import WKTElement from snowexsql.api import ( - PointMeasurements, LargeQueryCheckException, LayerMeasurements + PointMeasurements, LargeQueryCheckException, LayerMeasurements, db_session ) from snowexsql.db import get_db, initialize +from snowexsql.tables import Instrument, Observer, PointData @pytest.fixture(scope="session") @@ -45,8 +46,58 @@ def db(self, creds, db_url): metadata.drop_all(bind=engine) session.close() + @staticmethod + def _add_entry(url, instrument_name, observer_names, **kwargs): + url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" + with db_session(url_long) as (session, engine): + # Check if the instrument already exists + instrument = session.query(Instrument).filter_by( + name=instrument_name).first() + + if not instrument: + # If the instrument does not exist, create it + instrument = Instrument(name=instrument_name) + session.add(instrument) + session.commit() # Commit to ensure instrument is saved and has an ID + + observer_list = [] + for obs_name in observer_names: + observer = session.query(Observer).filter_by( + last_name=obs_name).first() + if not observer: + # If the instrument does not exist, create it + observer = Observer(last_name=obs_name) + session.add(observer) + session.commit() # Commit to ensure instrument is saved and has an ID + observer_list.append(observer) + + # Now that the instrument exists, create the entry, notice we only need the instrument object + new_entry = PointData( + instrument=instrument, observers=observer_list, **kwargs + ) + session.add(new_entry) + session.commit() + @pytest.fixture(scope="class") - def clz(self, db, db_url): + def populated_points(self, db): + + # Fake data to implement + row = { + 'date': date(2020, 1, 28), + 'time': time(18, 48), 'longitude': -108.13515, + 'latitude': 39.03045, + 'easting': 747987.6190615438, 'northing': 4324061.7062127385, + 'elevation': 3148.2, + 'equipment': 'CRREL_B', 'version_number': 1, 'utm_zone': 12, + 'geom': WKTElement("POINT(747987.6190615438 4324061.7062127385)", + srid=26912), + 'site_name': 'Grand Mesa', 'date_accessed': date(2024, 7, 10), + 'value': 94, 'type': 'depth', 'units': 'cm' + } + self._add_entry(db.url, 'magnaprobe', ["TEST"], **row) + + @pytest.fixture(scope="class") + def clz(self, db, db_url, populated_points): """ Extend the class and overwrite the database name """ @@ -57,10 +108,10 @@ class Extended(self.CLZ): yield Extended -def unsorted_list_tuple_compare(l1, l2): +def unsorted_list_compare(l1, l2): # turn lists into sets, but get rid of any Nones - l1 = set([l[0] for l in l1 if l[0] is not None]) - l2 = set([l[0] for l in l2 if l[0] is not None]) + l1 = set(l1) + l2 = set(l2) # compare the sets return l1 == l2 @@ -73,31 +124,31 @@ class TestPointMeasurements(DBConnection): def test_all_types(self, clz): result = clz().all_types - assert unsorted_list_tuple_compare( + assert unsorted_list_compare( result, - [] + ['depth'] ) def test_all_site_names(self, clz): result = clz().all_site_names - assert unsorted_list_tuple_compare( - result, [] + assert unsorted_list_compare( + result, ['Grand Mesa'] ) def test_all_dates(self, clz): result = clz().all_dates - assert len(result) == 0 + assert len(result) == 1 def test_all_observers(self, clz): result = clz().all_observers - assert unsorted_list_tuple_compare( - result, [] + assert unsorted_list_compare( + result, ['None TEST'] ) def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_tuple_compare( - result, [] + assert unsorted_list_compare( + result, ["magnaprobe"] ) @pytest.mark.parametrize( @@ -179,11 +230,11 @@ def test_all_dates(self, clz): def test_all_observers(self, clz): result = clz().all_observers - assert unsorted_list_tuple_compare(result, []) + assert unsorted_list_compare(result, []) def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_tuple_compare(result, []) + assert unsorted_list_compare(result, []) @pytest.mark.parametrize( "kwargs, expected_length, mean_value", [ From e40e2039bd9f22dfe4fd32e6e87cb34a22eb7f4e Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 22 Aug 2024 17:01:10 -0600 Subject: [PATCH 05/16] Get rid of observers table --- snowexsql/api.py | 17 +---------------- snowexsql/tables/__init__.py | 2 -- snowexsql/tables/observers.py | 14 -------------- snowexsql/tables/point_data.py | 19 ------------------- tests/test_api.py | 29 +++++++---------------------- 5 files changed, 8 insertions(+), 73 deletions(-) delete mode 100644 snowexsql/tables/observers.py diff --git a/snowexsql/api.py b/snowexsql/api.py index 6652737..6fa8a61 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -10,9 +10,7 @@ from snowexsql.conversions import query_to_geopandas, raster_to_rasterio from snowexsql.db import get_db -from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ - Observer -from snowexsql.tables.point_data import PointObservers +from snowexsql.tables import ImageData, LayerData, PointData, Instrument LOG = logging.getLogger(__name__) DB_NAME = 'snow:hackweek@db.snowexdata.org/snowex' @@ -201,19 +199,6 @@ def all_dates(self): result = qry.all() return self.retrieve_single_value_result(result) - @property - def all_observers(self): - """ - Return all distinct observers in the data - """ - with db_session(self.DB_NAME) as (session, engine): - qry = session.query(Observer).join( - PointObservers, Observer.id == PointObservers.observer_id - ).distinct() - result = qry.all() - # Join the names - return [f"{r.first_name} {r.last_name}" for r in result] - @property def all_units(self): """ diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index c778d55..b9f2e30 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -2,7 +2,6 @@ from .layer_data import LayerData from .point_data import PointData from .site_data import SiteData -from .observers import Observer from .instrument import Instrument __all__ = [ @@ -10,5 +9,4 @@ 'LayerData', 'PointData', "Instrument", - "Observer", ] diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py deleted file mode 100644 index 0cf59d0..0000000 --- a/snowexsql/tables/observers.py +++ /dev/null @@ -1,14 +0,0 @@ -from sqlalchemy.orm import mapped_column, Mapped -from sqlalchemy import Column, String - -from .base import Base - - -class Observer(Base): - __tablename__ = 'observers' - __table_args__ = {"schema": "public"} - # id is mapped column for many-to-many - id: Mapped[int] = mapped_column(primary_key=True) - # Name of the observer - first_name = Column(String(255)) - last_name = Column(String(255)) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index e421d43..5e90f28 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -5,22 +5,9 @@ from sqlalchemy.orm import relationship from .base import Base, Measurement, SingleLocationData -from .observers import Observer from .instrument import Instrument -class PointObservers(Base): - """ - Link table - """ - __tablename__ = 'point_observers' - __table_args__ = {'schema': 'public'} - - id = Column(Integer, primary_key=True) - point_id = Column(Integer, ForeignKey('public.points.id')) - observer_id = Column(Integer, ForeignKey("public.observers.id")) - - class PointData(SingleLocationData, Measurement, Base): """ Class representing the points table. This table holds all point data. @@ -41,9 +28,3 @@ class PointData(SingleLocationData, Measurement, Base): instrument_id = Column(Integer, ForeignKey('public.instruments.id')) # Link the Instrument class instrument = relationship('Instrument') - - # id is a mapped column for many-to-many with observers - id: Mapped[int] = mapped_column(primary_key=True) - observers: Mapped[List[Observer]] = relationship( - secondary=PointObservers.__table__ - ) diff --git a/tests/test_api.py b/tests/test_api.py index adcb0c9..a0b0d25 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,7 @@ PointMeasurements, LargeQueryCheckException, LayerMeasurements, db_session ) from snowexsql.db import get_db, initialize -from snowexsql.tables import Instrument, Observer, PointData +from snowexsql.tables import Instrument, PointData @pytest.fixture(scope="session") @@ -47,7 +47,7 @@ def db(self, creds, db_url): session.close() @staticmethod - def _add_entry(url, instrument_name, observer_names, **kwargs): + def _add_entry(url, instrument_name, **kwargs): url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" with db_session(url_long) as (session, engine): # Check if the instrument already exists @@ -60,20 +60,9 @@ def _add_entry(url, instrument_name, observer_names, **kwargs): session.add(instrument) session.commit() # Commit to ensure instrument is saved and has an ID - observer_list = [] - for obs_name in observer_names: - observer = session.query(Observer).filter_by( - last_name=obs_name).first() - if not observer: - # If the instrument does not exist, create it - observer = Observer(last_name=obs_name) - session.add(observer) - session.commit() # Commit to ensure instrument is saved and has an ID - observer_list.append(observer) - # Now that the instrument exists, create the entry, notice we only need the instrument object new_entry = PointData( - instrument=instrument, observers=observer_list, **kwargs + instrument=instrument, **kwargs ) session.add(new_entry) session.commit() @@ -94,7 +83,7 @@ def populated_points(self, db): 'site_name': 'Grand Mesa', 'date_accessed': date(2024, 7, 10), 'value': 94, 'type': 'depth', 'units': 'cm' } - self._add_entry(db.url, 'magnaprobe', ["TEST"], **row) + self._add_entry(db.url, 'magnaprobe', **row) @pytest.fixture(scope="class") def clz(self, db, db_url, populated_points): @@ -139,12 +128,6 @@ def test_all_dates(self, clz): result = clz().all_dates assert len(result) == 1 - def test_all_observers(self, clz): - result = clz().all_observers - assert unsorted_list_compare( - result, ['None TEST'] - ) - def test_all_instruments(self, clz): result = clz().all_instruments assert unsorted_list_compare( @@ -157,7 +140,9 @@ def test_all_instruments(self, clz): "date": date(2020, 5, 28), "instrument": 'camera' }, 0, np.nan), - ({"instrument": "magnaprobe", "limit": 10}, 0, np.nan), # limit works + ({ + "instrument": "magnaprobe", "limit": 10 + }, 1, 94.0), # limit works ({ "date": date(2020, 5, 28), "instrument": 'pit ruler' From 6ac68309a038222c277d15066ddb8b88538a01eb Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Fri, 23 Aug 2024 11:32:21 -0600 Subject: [PATCH 06/16] delete non-used data from test, get rid of length reqs on some strings --- snowexsql/tables/instrument.py | 6 +++--- snowexsql/tables/point_data.py | 10 ++++++---- tests/test_api.py | 7 +++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/snowexsql/tables/instrument.py b/snowexsql/tables/instrument.py index 9d40717..a4f2b21 100644 --- a/snowexsql/tables/instrument.py +++ b/snowexsql/tables/instrument.py @@ -12,6 +12,6 @@ class Instrument(Base): # auto created id id = Column(Integer, primary_key=True) # Name of the instrument - name = Column(String(50)) - model = Column(String(255)) - specifications = Column(String(255)) + name = Column(String(), index=True) + model = Column(String()) + specifications = Column(String()) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index 5e90f28..60907f6 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -17,14 +17,16 @@ class PointData(SingleLocationData, Measurement, Base): __tablename__ = 'points' version_number = Column(Integer) - equipment = Column(String(50)) + equipment = Column(String()) value = Column(Float) # bring these in instead of Measurement - type = Column(String(50)) - units = Column(String(50)) + type = Column(String()) + units = Column(String()) # Link the instrument id with a foreign key - instrument_id = Column(Integer, ForeignKey('public.instruments.id')) + instrument_id = Column( + Integer, ForeignKey('public.instruments.id'), index=True + ) # Link the Instrument class instrument = relationship('Instrument') diff --git a/tests/test_api.py b/tests/test_api.py index a0b0d25..5549db7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -73,11 +73,10 @@ def populated_points(self, db): # Fake data to implement row = { 'date': date(2020, 1, 28), - 'time': time(18, 48), 'longitude': -108.13515, - 'latitude': 39.03045, - 'easting': 747987.6190615438, 'northing': 4324061.7062127385, + 'time': time(18, 48), 'elevation': 3148.2, - 'equipment': 'CRREL_B', 'version_number': 1, 'utm_zone': 12, + 'equipment': 'CRREL_B', + 'version_number': 1, 'geom': WKTElement("POINT(747987.6190615438 4324061.7062127385)", srid=26912), 'site_name': 'Grand Mesa', 'date_accessed': date(2024, 7, 10), From da73962b28bbccd011681f1e7b0bbbc8ca92f5a0 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Fri, 23 Aug 2024 12:30:29 -0600 Subject: [PATCH 07/16] Revert "Get rid of observers table" This reverts commit e40e2039bd9f22dfe4fd32e6e87cb34a22eb7f4e. --- snowexsql/api.py | 17 ++++++++++++++++- snowexsql/tables/__init__.py | 2 ++ snowexsql/tables/observers.py | 14 ++++++++++++++ snowexsql/tables/point_data.py | 19 +++++++++++++++++++ tests/test_api.py | 29 ++++++++++++++++++++++------- 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 snowexsql/tables/observers.py diff --git a/snowexsql/api.py b/snowexsql/api.py index 6fa8a61..6652737 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -10,7 +10,9 @@ from snowexsql.conversions import query_to_geopandas, raster_to_rasterio from snowexsql.db import get_db -from snowexsql.tables import ImageData, LayerData, PointData, Instrument +from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ + Observer +from snowexsql.tables.point_data import PointObservers LOG = logging.getLogger(__name__) DB_NAME = 'snow:hackweek@db.snowexdata.org/snowex' @@ -199,6 +201,19 @@ def all_dates(self): result = qry.all() return self.retrieve_single_value_result(result) + @property + def all_observers(self): + """ + Return all distinct observers in the data + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(Observer).join( + PointObservers, Observer.id == PointObservers.observer_id + ).distinct() + result = qry.all() + # Join the names + return [f"{r.first_name} {r.last_name}" for r in result] + @property def all_units(self): """ diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index b9f2e30..c778d55 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -2,6 +2,7 @@ from .layer_data import LayerData from .point_data import PointData from .site_data import SiteData +from .observers import Observer from .instrument import Instrument __all__ = [ @@ -9,4 +10,5 @@ 'LayerData', 'PointData', "Instrument", + "Observer", ] diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py new file mode 100644 index 0000000..0cf59d0 --- /dev/null +++ b/snowexsql/tables/observers.py @@ -0,0 +1,14 @@ +from sqlalchemy.orm import mapped_column, Mapped +from sqlalchemy import Column, String + +from .base import Base + + +class Observer(Base): + __tablename__ = 'observers' + __table_args__ = {"schema": "public"} + # id is mapped column for many-to-many + id: Mapped[int] = mapped_column(primary_key=True) + # Name of the observer + first_name = Column(String(255)) + last_name = Column(String(255)) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index 60907f6..d3c67fc 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -5,9 +5,22 @@ from sqlalchemy.orm import relationship from .base import Base, Measurement, SingleLocationData +from .observers import Observer from .instrument import Instrument +class PointObservers(Base): + """ + Link table + """ + __tablename__ = 'point_observers' + __table_args__ = {'schema': 'public'} + + id = Column(Integer, primary_key=True) + point_id = Column(Integer, ForeignKey('public.points.id')) + observer_id = Column(Integer, ForeignKey("public.observers.id")) + + class PointData(SingleLocationData, Measurement, Base): """ Class representing the points table. This table holds all point data. @@ -30,3 +43,9 @@ class PointData(SingleLocationData, Measurement, Base): ) # Link the Instrument class instrument = relationship('Instrument') + + # id is a mapped column for many-to-many with observers + id: Mapped[int] = mapped_column(primary_key=True) + observers: Mapped[List[Observer]] = relationship( + secondary=PointObservers.__table__ + ) diff --git a/tests/test_api.py b/tests/test_api.py index 5549db7..0fe6b75 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,7 @@ PointMeasurements, LargeQueryCheckException, LayerMeasurements, db_session ) from snowexsql.db import get_db, initialize -from snowexsql.tables import Instrument, PointData +from snowexsql.tables import Instrument, Observer, PointData @pytest.fixture(scope="session") @@ -47,7 +47,7 @@ def db(self, creds, db_url): session.close() @staticmethod - def _add_entry(url, instrument_name, **kwargs): + def _add_entry(url, instrument_name, observer_names, **kwargs): url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" with db_session(url_long) as (session, engine): # Check if the instrument already exists @@ -60,9 +60,20 @@ def _add_entry(url, instrument_name, **kwargs): session.add(instrument) session.commit() # Commit to ensure instrument is saved and has an ID + observer_list = [] + for obs_name in observer_names: + observer = session.query(Observer).filter_by( + last_name=obs_name).first() + if not observer: + # If the instrument does not exist, create it + observer = Observer(last_name=obs_name) + session.add(observer) + session.commit() # Commit to ensure instrument is saved and has an ID + observer_list.append(observer) + # Now that the instrument exists, create the entry, notice we only need the instrument object new_entry = PointData( - instrument=instrument, **kwargs + instrument=instrument, observers=observer_list, **kwargs ) session.add(new_entry) session.commit() @@ -82,7 +93,7 @@ def populated_points(self, db): 'site_name': 'Grand Mesa', 'date_accessed': date(2024, 7, 10), 'value': 94, 'type': 'depth', 'units': 'cm' } - self._add_entry(db.url, 'magnaprobe', **row) + self._add_entry(db.url, 'magnaprobe', ["TEST"], **row) @pytest.fixture(scope="class") def clz(self, db, db_url, populated_points): @@ -127,6 +138,12 @@ def test_all_dates(self, clz): result = clz().all_dates assert len(result) == 1 + def test_all_observers(self, clz): + result = clz().all_observers + assert unsorted_list_compare( + result, ['None TEST'] + ) + def test_all_instruments(self, clz): result = clz().all_instruments assert unsorted_list_compare( @@ -139,9 +156,7 @@ def test_all_instruments(self, clz): "date": date(2020, 5, 28), "instrument": 'camera' }, 0, np.nan), - ({ - "instrument": "magnaprobe", "limit": 10 - }, 1, 94.0), # limit works + ({"instrument": "magnaprobe", "limit": 10}, 0, np.nan), # limit works ({ "date": date(2020, 5, 28), "instrument": 'pit ruler' From 63492fede651505e0e43c7eb9511ea909959ee53 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Fri, 23 Aug 2024 12:36:00 -0600 Subject: [PATCH 08/16] Add observers back in --- snowexsql/api.py | 2 +- snowexsql/tables/observers.py | 3 +-- tests/test_api.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/snowexsql/api.py b/snowexsql/api.py index 6652737..278fa67 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -212,7 +212,7 @@ def all_observers(self): ).distinct() result = qry.all() # Join the names - return [f"{r.first_name} {r.last_name}" for r in result] + return [r.name for r in result] @property def all_units(self): diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py index 0cf59d0..8b42b9d 100644 --- a/snowexsql/tables/observers.py +++ b/snowexsql/tables/observers.py @@ -10,5 +10,4 @@ class Observer(Base): # id is mapped column for many-to-many id: Mapped[int] = mapped_column(primary_key=True) # Name of the observer - first_name = Column(String(255)) - last_name = Column(String(255)) + name = Column(String()) diff --git a/tests/test_api.py b/tests/test_api.py index 0fe6b75..24ca936 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -63,10 +63,10 @@ def _add_entry(url, instrument_name, observer_names, **kwargs): observer_list = [] for obs_name in observer_names: observer = session.query(Observer).filter_by( - last_name=obs_name).first() + name=obs_name).first() if not observer: # If the instrument does not exist, create it - observer = Observer(last_name=obs_name) + observer = Observer(name=obs_name) session.add(observer) session.commit() # Commit to ensure instrument is saved and has an ID observer_list.append(observer) @@ -141,7 +141,7 @@ def test_all_dates(self, clz): def test_all_observers(self, clz): result = clz().all_observers assert unsorted_list_compare( - result, ['None TEST'] + result, ['TEST'] ) def test_all_instruments(self, clz): @@ -156,7 +156,7 @@ def test_all_instruments(self, clz): "date": date(2020, 5, 28), "instrument": 'camera' }, 0, np.nan), - ({"instrument": "magnaprobe", "limit": 10}, 0, np.nan), # limit works + ({"instrument": "magnaprobe", "limit": 10}, 1, 94.0), # limit works ({ "date": date(2020, 5, 28), "instrument": 'pit ruler' From 9d12cfeca3323836fcd96454467bc2df09894382 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Fri, 23 Aug 2024 19:19:33 -0600 Subject: [PATCH 09/16] Add campaigns link --- snowexsql/api.py | 11 +++++--- .../tables/{campaign_data.py => campaign.py} | 13 ++++----- snowexsql/tables/point_data.py | 8 ++++++ tests/test_api.py | 27 +++++++++++++++---- 4 files changed, 43 insertions(+), 16 deletions(-) rename snowexsql/tables/{campaign_data.py => campaign.py} (64%) diff --git a/snowexsql/api.py b/snowexsql/api.py index 278fa67..b9b79f4 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -12,6 +12,7 @@ from snowexsql.db import get_db from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ Observer +from snowexsql.tables.campaign import Campaign from snowexsql.tables.point_data import PointObservers LOG = logging.getLogger(__name__) @@ -46,6 +47,7 @@ def get_points(): class BaseDataset: MODEL = None + LINK_TABLE_MODEL = PointObservers # Use this database name DB_NAME = DB_NAME @@ -177,7 +179,9 @@ def all_site_names(self): Return all types of the data """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(self.MODEL.site_name).distinct() + qry = session.query(Campaign.name).join( + self.MODEL, Campaign.id == self.MODEL.campaign_id + ).distinct() result = qry.all() return self.retrieve_single_value_result(result) @@ -208,7 +212,8 @@ def all_observers(self): """ with db_session(self.DB_NAME) as (session, engine): qry = session.query(Observer).join( - PointObservers, Observer.id == PointObservers.observer_id + self.LINK_TABLE_MODEL, + Observer.id == self.LINK_TABLE_MODEL.observer_id ).distinct() result = qry.all() # Join the names @@ -231,7 +236,7 @@ def all_instruments(self): """ with db_session(self.DB_NAME) as (session, engine): qry = session.query(Instrument.name).join( - PointData, Instrument.id == PointData.instrument_id + self.MODEL, Instrument.id == self.MODEL.instrument_id ).distinct() result = qry.all() return self.retrieve_single_value_result(result) diff --git a/snowexsql/tables/campaign_data.py b/snowexsql/tables/campaign.py similarity index 64% rename from snowexsql/tables/campaign_data.py rename to snowexsql/tables/campaign.py index b56988e..f002e91 100644 --- a/snowexsql/tables/campaign_data.py +++ b/snowexsql/tables/campaign.py @@ -10,16 +10,13 @@ from .base import Base -class CampaignData(Base): +class Campaign(Base): """ Table stores Campaign data. Does not store data values, it only stores the campaign metadata. """ - __tablename__ = 'sites' - __table_args__ = {"schema": "public"} - + __tablename__ = 'campaigns' + id = Column(Integer, primary_key=True) - name = Column(String(100)) - description = Column(String(1000)) - - \ No newline at end of file + name = Column(String()) + description = Column(String()) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index d3c67fc..f3e9fc3 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -7,6 +7,7 @@ from .base import Base, Measurement, SingleLocationData from .observers import Observer from .instrument import Instrument +from .campaign import Campaign class PointObservers(Base): @@ -44,6 +45,13 @@ class PointData(SingleLocationData, Measurement, Base): # Link the Instrument class instrument = relationship('Instrument') + # Link the campaign id with a foreign key + campaign_id = Column( + Integer, ForeignKey('public.campaigns.id'), index=True + ) + # Link the Campaign class + campaign = relationship('Campaign') + # id is a mapped column for many-to-many with observers id: Mapped[int] = mapped_column(primary_key=True) observers: Mapped[List[Observer]] = relationship( diff --git a/tests/test_api.py b/tests/test_api.py index 24ca936..890db02 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -9,6 +9,7 @@ ) from snowexsql.db import get_db, initialize from snowexsql.tables import Instrument, Observer, PointData +from snowexsql.tables.campaign import Campaign @pytest.fixture(scope="session") @@ -36,7 +37,8 @@ class DBConnection: @pytest.fixture(scope="class") def db(self, creds, db_url): engine, session, metadata = get_db( - db_url, credentials=creds, return_metadata=True) + db_url, credentials=creds, return_metadata=True + ) initialize(engine) yield engine @@ -47,7 +49,9 @@ def db(self, creds, db_url): session.close() @staticmethod - def _add_entry(url, instrument_name, observer_names, **kwargs): + def _add_entry( + url, instrument_name, observer_names, campaign_name, **kwargs + ): url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" with db_session(url_long) as (session, engine): # Check if the instrument already exists @@ -60,6 +64,15 @@ def _add_entry(url, instrument_name, observer_names, **kwargs): session.add(instrument) session.commit() # Commit to ensure instrument is saved and has an ID + campaign = session.query(Campaign).filter_by( + name=campaign_name).first() + + if not campaign: + # If the campaign does not exist, create it + campaign = Campaign(name=campaign_name) + session.add(campaign) + session.commit() # Commit to ensure instrument is saved and has an ID + observer_list = [] for obs_name in observer_names: observer = session.query(Observer).filter_by( @@ -73,7 +86,8 @@ def _add_entry(url, instrument_name, observer_names, **kwargs): # Now that the instrument exists, create the entry, notice we only need the instrument object new_entry = PointData( - instrument=instrument, observers=observer_list, **kwargs + instrument=instrument, observers=observer_list, + campaign=campaign, **kwargs ) session.add(new_entry) session.commit() @@ -90,10 +104,13 @@ def populated_points(self, db): 'version_number': 1, 'geom': WKTElement("POINT(747987.6190615438 4324061.7062127385)", srid=26912), - 'site_name': 'Grand Mesa', 'date_accessed': date(2024, 7, 10), + 'date_accessed': date(2024, 7, 10), 'value': 94, 'type': 'depth', 'units': 'cm' } - self._add_entry(db.url, 'magnaprobe', ["TEST"], **row) + self._add_entry( + db.url, 'magnaprobe', ["TEST"], + 'Grand Mesa', **row + ) @pytest.fixture(scope="class") def clz(self, db, db_url, populated_points): From 8e557728341a0fc10d9ef2fa0c53c90e3598ecb5 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 11 Sep 2024 16:26:54 -0600 Subject: [PATCH 10/16] Layer data tests working for test_api --- snowexsql/tables/base.py | 1 - snowexsql/tables/layer_data.py | 41 +++++++++++++++++++++++++++++++++- tests/test_api.py | 41 +++++++++++++++++++++++++--------- tests/test_db.py | 2 +- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/snowexsql/tables/base.py b/snowexsql/tables/base.py index e0772a4..fc3df01 100644 --- a/snowexsql/tables/base.py +++ b/snowexsql/tables/base.py @@ -26,7 +26,6 @@ class Base(DeclarativeBase): time_updated = Column(DateTime(timezone=True), onupdate=func.now()) date_accessed = Column(Date) - site_name = Column(String(250)) date = Column(Date) doi = Column(String(50)) diff --git a/snowexsql/tables/layer_data.py b/snowexsql/tables/layer_data.py index 603da42..436af69 100644 --- a/snowexsql/tables/layer_data.py +++ b/snowexsql/tables/layer_data.py @@ -1,6 +1,25 @@ -from sqlalchemy import Column, Float, String +from sqlalchemy import Column, Float, Integer, String, ForeignKey +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column +from typing import List +from sqlalchemy.orm import relationship from .base import Base, Measurement, SingleLocationData +from .observers import Observer +from .instrument import Instrument +from .campaign import Campaign + + +class LayerObservers(Base): + """ + Link table + """ + __tablename__ = 'layer_observers' + __table_args__ = {'schema': 'public'} + + id = Column(Integer, primary_key=True) + layer_id = Column(Integer, ForeignKey('public.layers.id')) + observer_id = Column(Integer, ForeignKey("public.observers.id")) class LayerData(SingleLocationData, Measurement, Base): @@ -22,3 +41,23 @@ class LayerData(SingleLocationData, Measurement, Base): sample_c = Column(String(20)) value = Column(String(50)) flags = Column(String(20)) + + # Link the instrument id with a foreign key + instrument_id = Column( + Integer, ForeignKey('public.instruments.id'), index=True + ) + # Link the Instrument class + instrument = relationship('Instrument') + + # Link the campaign id with a foreign key + campaign_id = Column( + Integer, ForeignKey('public.campaigns.id'), index=True + ) + # Link the Campaign class + campaign = relationship('Campaign') + + # id is a mapped column for many-to-many with observers + id: Mapped[int] = mapped_column(primary_key=True) + observers: Mapped[List[Observer]] = relationship( + secondary=LayerObservers.__table__ + ) diff --git a/tests/test_api.py b/tests/test_api.py index 890db02..8efcfc9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,7 @@ PointMeasurements, LargeQueryCheckException, LayerMeasurements, db_session ) from snowexsql.db import get_db, initialize -from snowexsql.tables import Instrument, Observer, PointData +from snowexsql.tables import Instrument, Observer, PointData, LayerData from snowexsql.tables.campaign import Campaign @@ -50,7 +50,8 @@ def db(self, creds, db_url): @staticmethod def _add_entry( - url, instrument_name, observer_names, campaign_name, **kwargs + url, data_cls, instrument_name, observer_names, campaign_name, + **kwargs ): url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" with db_session(url_long) as (session, engine): @@ -85,7 +86,7 @@ def _add_entry( observer_list.append(observer) # Now that the instrument exists, create the entry, notice we only need the instrument object - new_entry = PointData( + new_entry = data_cls( instrument=instrument, observers=observer_list, campaign=campaign, **kwargs ) @@ -108,12 +109,32 @@ def populated_points(self, db): 'value': 94, 'type': 'depth', 'units': 'cm' } self._add_entry( - db.url, 'magnaprobe', ["TEST"], + db.url, PointData, 'magnaprobe', ["TEST"], 'Grand Mesa', **row ) @pytest.fixture(scope="class") - def clz(self, db, db_url, populated_points): + def populated_layer(self, db): + + # Fake data to implement + row = { + 'date': date(2020, 1, 28), + 'time': time(18, 48), + 'elevation': 3148.2, + 'geom': WKTElement("POINT(747987.6190615438 4324061.7062127385)", + srid=26912), + 'date_accessed': date(2024, 7, 10), + 'value': '42.5', 'type': 'Density', 'units': 'kgm3', + 'pit_id': 'Fakepit1', + 'sample_a': '42.5' + } + self._add_entry( + db.url, LayerData, 'magnaprobe', ["TEST"], + 'Grand Mesa', **row + ) + + @pytest.fixture(scope="class") + def clz(self, db, db_url, populated_points, populated_layer): """ Extend the class and overwrite the database name """ @@ -234,23 +255,23 @@ class TestLayerMeasurements(DBConnection): def test_all_types(self, clz): result = clz().all_types - assert result == [] + assert result == ["Density"] def test_all_site_names(self, clz): result = clz().all_site_names - assert result == [] + assert result == ['Grand Mesa'] def test_all_dates(self, clz): result = clz().all_dates - assert len(result) == 0 + assert result == [date(2020, 1, 28)] def test_all_observers(self, clz): result = clz().all_observers - assert unsorted_list_compare(result, []) + assert unsorted_list_compare(result, ['TEST']) def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_compare(result, []) + assert unsorted_list_compare(result, ['magnaprobe']) @pytest.mark.parametrize( "kwargs, expected_length, mean_value", [ diff --git a/tests/test_db.py b/tests/test_db.py index 35b979d..c7624ba 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -9,7 +9,7 @@ class TestDB(DBSetup): - base_atts = ['site_name', 'date', 'site_id'] + base_atts = ['date', 'site_id'] single_loc_atts = ['elevation', 'geom', 'time'] meas_atts = ['type', 'units'] From 79edfd73596ca30fc69ce632abebd52a66dd7836 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Wed, 11 Sep 2024 16:42:08 -0600 Subject: [PATCH 11/16] link table spec --- snowexsql/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snowexsql/api.py b/snowexsql/api.py index b9b79f4..b4c7bd6 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -13,6 +13,7 @@ from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ Observer from snowexsql.tables.campaign import Campaign +from snowexsql.tables.layer_data import LayerObservers from snowexsql.tables.point_data import PointObservers LOG = logging.getLogger(__name__) @@ -331,6 +332,7 @@ class LayerMeasurements(PointMeasurements): "site_name", "site_id", "date", "instrument", "observers", "type", "utm_zone", "pit_id", "date_greater_equal", "date_less_equal" ] + LINK_TABLE_MODEL = LayerObservers # TODO: layer analysis methods? @property From aa2654a0242233e616823975d80e2bca87d83e7e Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 12 Sep 2024 08:39:08 -0600 Subject: [PATCH 12/16] Filtering by campaign and SINGLE observer, including tests --- snowexsql/api.py | 15 ++++++++++++--- tests/test_api.py | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/snowexsql/api.py b/snowexsql/api.py index b4c7bd6..c230a8e 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -53,7 +53,7 @@ class BaseDataset: DB_NAME = DB_NAME ALLOWED_QRY_KWARGS = [ - "site_name", "site_id", "date", "instrument", "type", + "campaign", "site_id", "date", "instrument", "type", "utm_zone", "date_greater_equal", "date_less_equal", "value_greater_equal", 'value_less_equal', ] @@ -131,6 +131,14 @@ def extend_qry(cls, qry, check_size=True, **kwargs): qry = qry.filter( cls.MODEL.instrument.has(name=v) ) + elif k == "campaign": + qry = qry.filter( + cls.MODEL.campaign.has(name=v) + ) + elif k == "observer": + qry = qry.join( + LayerData.observers + ).filter(Observer.name == v) # Filter to exact value else: filter_col = getattr(cls.MODEL, k) @@ -319,21 +327,22 @@ def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs): return df + class TooManyRastersException(Exception): """ Exceptiont to report to users that their query will produce too many rasters""" pass + class LayerMeasurements(PointMeasurements): """ API class for access to LayerData """ MODEL = LayerData ALLOWED_QRY_KWARGS = [ - "site_name", "site_id", "date", "instrument", "observers", "type", + "campaign", "site_id", "date", "instrument", "observer", "type", "utm_zone", "pit_id", "date_greater_equal", "date_less_equal" ] LINK_TABLE_MODEL = LayerObservers - # TODO: layer analysis methods? @property def all_site_ids(self): diff --git a/tests/test_api.py b/tests/test_api.py index 8efcfc9..8f4ef81 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -124,12 +124,12 @@ def populated_layer(self, db): 'geom': WKTElement("POINT(747987.6190615438 4324061.7062127385)", srid=26912), 'date_accessed': date(2024, 7, 10), - 'value': '42.5', 'type': 'Density', 'units': 'kgm3', + 'value': '42.5', 'type': 'density', 'units': 'kgm3', 'pit_id': 'Fakepit1', 'sample_a': '42.5' } self._add_entry( - db.url, LayerData, 'magnaprobe', ["TEST"], + db.url, LayerData, 'fakeinstrument', ["TEST"], 'Grand Mesa', **row ) @@ -255,7 +255,7 @@ class TestLayerMeasurements(DBConnection): def test_all_types(self, clz): result = clz().all_types - assert result == ["Density"] + assert result == ["density"] def test_all_site_names(self, clz): result = clz().all_site_names @@ -271,7 +271,7 @@ def test_all_observers(self, clz): def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_compare(result, ['magnaprobe']) + assert unsorted_list_compare(result, ['fakeinstrument']) @pytest.mark.parametrize( "kwargs, expected_length, mean_value", [ @@ -292,6 +292,14 @@ def test_all_instruments(self, clz): "date_greater_equal": date(2020, 5, 13), "type": 'density' }, 0, np.nan), + ({ + "type": 'density', + "campaign": 'Grand Mesa' + }, 1, 42.5), + ({ + "observer": 'TEST', + "campaign": 'Grand Mesa' + }, 1, 42.5), ] ) def test_from_filter(self, clz, kwargs, expected_length, mean_value): From aa16b7a3f48c822baa55a8329c21f773dc41a803 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 12 Sep 2024 14:46:44 -0600 Subject: [PATCH 13/16] Addressing PR feedback --- snowexsql/tables/__init__.py | 8 +++++--- snowexsql/tables/instrument.py | 5 ----- snowexsql/tables/observers.py | 1 - snowexsql/tables/point_data.py | 1 - snowexsql/tables/site_data.py | 1 - tests/sql_test_base.py | 2 +- tests/test_api.py | 32 +++++++------------------------- 7 files changed, 13 insertions(+), 37 deletions(-) diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index c778d55..68b0366 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -4,11 +4,13 @@ from .site_data import SiteData from .observers import Observer from .instrument import Instrument +from .campaign import Campaign __all__ = [ - 'ImageData', - 'LayerData', - 'PointData', + "Campaign", + "ImageData", "Instrument", + "LayerData", "Observer", + "PointData", ] diff --git a/snowexsql/tables/instrument.py b/snowexsql/tables/instrument.py index a4f2b21..82a888c 100644 --- a/snowexsql/tables/instrument.py +++ b/snowexsql/tables/instrument.py @@ -4,11 +4,6 @@ class Instrument(Base): __tablename__ = 'instruments' - __table_args__ = {"schema": "public"} - # this seems important - __mapper_args__ = { - 'polymorphic_identity': 'Instruments', - } # auto created id id = Column(Integer, primary_key=True) # Name of the instrument diff --git a/snowexsql/tables/observers.py b/snowexsql/tables/observers.py index 8b42b9d..07f1e0e 100644 --- a/snowexsql/tables/observers.py +++ b/snowexsql/tables/observers.py @@ -6,7 +6,6 @@ class Observer(Base): __tablename__ = 'observers' - __table_args__ = {"schema": "public"} # id is mapped column for many-to-many id: Mapped[int] = mapped_column(primary_key=True) # Name of the observer diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index f3e9fc3..01e19da 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -15,7 +15,6 @@ class PointObservers(Base): Link table """ __tablename__ = 'point_observers' - __table_args__ = {'schema': 'public'} id = Column(Integer, primary_key=True) point_id = Column(Integer, ForeignKey('public.points.id')) diff --git a/snowexsql/tables/site_data.py b/snowexsql/tables/site_data.py index a88441b..5f1d1b0 100644 --- a/snowexsql/tables/site_data.py +++ b/snowexsql/tables/site_data.py @@ -9,7 +9,6 @@ class SiteData(SingleLocationData, Base): main data record but only support data for each site """ __tablename__ = 'sites' - __table_args__ = {"schema": "public"} pit_id = Column(String(50)) slope_angle = Column(Float) diff --git a/tests/sql_test_base.py b/tests/sql_test_base.py index 5222952..303e058 100644 --- a/tests/sql_test_base.py +++ b/tests/sql_test_base.py @@ -1,7 +1,7 @@ from os.path import dirname, join from numpy.testing import assert_almost_equal -from sqlalchemy import asc, text +from sqlalchemy import asc from snowexsql.db import get_db, initialize diff --git a/tests/test_api.py b/tests/test_api.py index 8f4ef81..b896f82 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -95,8 +95,7 @@ def _add_entry( @pytest.fixture(scope="class") def populated_points(self, db): - - # Fake data to implement + # Add made up data at the initialization of the class row = { 'date': date(2020, 1, 28), 'time': time(18, 48), @@ -145,14 +144,6 @@ class Extended(self.CLZ): yield Extended -def unsorted_list_compare(l1, l2): - # turn lists into sets, but get rid of any Nones - l1 = set(l1) - l2 = set(l2) - # compare the sets - return l1 == l2 - - class TestPointMeasurements(DBConnection): """ Test the Point Measurement class @@ -161,16 +152,11 @@ class TestPointMeasurements(DBConnection): def test_all_types(self, clz): result = clz().all_types - assert unsorted_list_compare( - result, - ['depth'] - ) + assert result == ['depth'] def test_all_site_names(self, clz): result = clz().all_site_names - assert unsorted_list_compare( - result, ['Grand Mesa'] - ) + assert result ==['Grand Mesa'] def test_all_dates(self, clz): result = clz().all_dates @@ -178,15 +164,11 @@ def test_all_dates(self, clz): def test_all_observers(self, clz): result = clz().all_observers - assert unsorted_list_compare( - result, ['TEST'] - ) + assert result == ['TEST'] def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_compare( - result, ["magnaprobe"] - ) + assert result == ["magnaprobe"] @pytest.mark.parametrize( "kwargs, expected_length, mean_value", [ @@ -267,11 +249,11 @@ def test_all_dates(self, clz): def test_all_observers(self, clz): result = clz().all_observers - assert unsorted_list_compare(result, ['TEST']) + assert result == ['TEST'] def test_all_instruments(self, clz): result = clz().all_instruments - assert unsorted_list_compare(result, ['fakeinstrument']) + assert result == ['fakeinstrument'] @pytest.mark.parametrize( "kwargs, expected_length, mean_value", [ From 61449051b2fdf4ca0947acde3e2ac675fe7b20f7 Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Thu, 12 Sep 2024 15:23:49 -0600 Subject: [PATCH 14/16] Add a campaign_sites table, adjust tests --- snowexsql/api.py | 28 +++++++++++++++++++++------- snowexsql/tables/__init__.py | 4 +++- snowexsql/tables/base.py | 1 - snowexsql/tables/layer_data.py | 13 ++++++------- snowexsql/tables/point_data.py | 12 ++++++------ snowexsql/tables/site.py | 31 +++++++++++++++++++++++++++++++ tests/test_api.py | 24 ++++++++++++++++++------ 7 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 snowexsql/tables/site.py diff --git a/snowexsql/api.py b/snowexsql/api.py index c230a8e..f1c4873 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -11,7 +11,7 @@ from snowexsql.conversions import query_to_geopandas, raster_to_rasterio from snowexsql.db import get_db from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ - Observer + Observer, Site from snowexsql.tables.campaign import Campaign from snowexsql.tables.layer_data import LayerObservers from snowexsql.tables.point_data import PointObservers @@ -132,8 +132,14 @@ def extend_qry(cls, qry, check_size=True, **kwargs): cls.MODEL.instrument.has(name=v) ) elif k == "campaign": + qry = qry.join( + cls.MODEL.site + ).filter( + Site.campaign.has(Campaign.name == v) + ) + elif k == "site_id": qry = qry.filter( - cls.MODEL.campaign.has(name=v) + cls.MODEL.site.has(name=v) ) elif k == "observer": qry = qry.join( @@ -185,12 +191,17 @@ def from_unique_entries(cls, columns_to_search, **kwargs): @property def all_site_names(self): """ - Return all types of the data + Return all campaign names """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(Campaign.name).join( - self.MODEL, Campaign.id == self.MODEL.campaign_id - ).distinct() + qry = ( + session.query(Campaign.name) # Selecting Campaign names + .join(Site, + Site.campaign_id == Campaign.id) # Join Site to Campaign + .join(self.MODEL, + self.MODEL.site_id == Site.id) + .distinct() # To get distinct Campaign names + ) result = qry.all() return self.retrieve_single_value_result(result) @@ -350,10 +361,13 @@ def all_site_ids(self): Return all types of the data """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(self.MODEL.site_id).distinct() + qry = session.query(Site.name).join( + self.MODEL, Site.id == self.MODEL.site_id + ).distinct() result = qry.all() return self.retrieve_single_value_result(result) + class RasterMeasurements(BaseDataset): MODEL = ImageData ALLOWED_QRY_KWARGS = BaseDataset.ALLOWED_QRY_KWARGS + ['description'] diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index 68b0366..b188446 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -5,6 +5,7 @@ from .observers import Observer from .instrument import Instrument from .campaign import Campaign +from .site import Site __all__ = [ "Campaign", @@ -12,5 +13,6 @@ "Instrument", "LayerData", "Observer", - "PointData", + "PointData", + "Site", ] diff --git a/snowexsql/tables/base.py b/snowexsql/tables/base.py index fc3df01..b946803 100644 --- a/snowexsql/tables/base.py +++ b/snowexsql/tables/base.py @@ -37,7 +37,6 @@ class SingleLocationData: elevation = Column(Float) geom = Column(Geometry("POINT")) time = Column(Time(timezone=True)) - site_id = Column(String(50)) class Measurement(object): diff --git a/snowexsql/tables/layer_data.py b/snowexsql/tables/layer_data.py index 436af69..6a4e0fd 100644 --- a/snowexsql/tables/layer_data.py +++ b/snowexsql/tables/layer_data.py @@ -7,7 +7,7 @@ from .base import Base, Measurement, SingleLocationData from .observers import Observer from .instrument import Instrument -from .campaign import Campaign +from .site import Site class LayerObservers(Base): @@ -32,7 +32,6 @@ class LayerData(SingleLocationData, Measurement, Base): __tablename__ = 'layers' depth = Column(Float) - site_id = Column(String(50)) pit_id = Column(String(50)) bottom_depth = Column(Float) comments = Column(String(1000)) @@ -49,12 +48,12 @@ class LayerData(SingleLocationData, Measurement, Base): # Link the Instrument class instrument = relationship('Instrument') - # Link the campaign id with a foreign key - campaign_id = Column( - Integer, ForeignKey('public.campaigns.id'), index=True + # Link the site id with a foreign key + site_id = Column( + Integer, ForeignKey('public.campaign_sites.id'), index=True ) - # Link the Campaign class - campaign = relationship('Campaign') + # Link the Site class + site = relationship('Site') # id is a mapped column for many-to-many with observers id: Mapped[int] = mapped_column(primary_key=True) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index 01e19da..9607611 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -7,7 +7,7 @@ from .base import Base, Measurement, SingleLocationData from .observers import Observer from .instrument import Instrument -from .campaign import Campaign +from .site import Site class PointObservers(Base): @@ -44,12 +44,12 @@ class PointData(SingleLocationData, Measurement, Base): # Link the Instrument class instrument = relationship('Instrument') - # Link the campaign id with a foreign key - campaign_id = Column( - Integer, ForeignKey('public.campaigns.id'), index=True + # Link the site id with a foreign key + site_id = Column( + Integer, ForeignKey('public.campaign_sites.id'), index=True ) - # Link the Campaign class - campaign = relationship('Campaign') + # Link the Site class + site = relationship('Site') # id is a mapped column for many-to-many with observers id: Mapped[int] = mapped_column(primary_key=True) diff --git a/snowexsql/tables/site.py b/snowexsql/tables/site.py new file mode 100644 index 0000000..4dce286 --- /dev/null +++ b/snowexsql/tables/site.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Aug 22 11:56:34 2024 + +@author: jtmaz +""" + +from sqlalchemy import Column, String, Integer, ForeignKey +from sqlalchemy.orm import relationship + +from .base import Base +from .campaign import Campaign + + +class Site(Base): + """ + Table stores Site data. Does not store data values, + it only stores the site metadata. + """ + __tablename__ = 'campaign_sites' + + id = Column(Integer, primary_key=True) + name = Column(String()) + description = Column(String()) + + # Link the campaign id with a foreign key + campaign_id = Column( + Integer, ForeignKey('public.campaigns.id'), index=True + ) + # Link the Campaign class + campaign = relationship('Campaign') diff --git a/tests/test_api.py b/tests/test_api.py index b896f82..68ea1d2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,7 @@ PointMeasurements, LargeQueryCheckException, LayerMeasurements, db_session ) from snowexsql.db import get_db, initialize -from snowexsql.tables import Instrument, Observer, PointData, LayerData +from snowexsql.tables import Instrument, Observer, PointData, LayerData, Site from snowexsql.tables.campaign import Campaign @@ -50,7 +50,8 @@ def db(self, creds, db_url): @staticmethod def _add_entry( - url, data_cls, instrument_name, observer_names, campaign_name, + url, data_cls, instrument_name, + observer_names, campaign_name, site_name, **kwargs ): url_long = f"{url.username}:{url.password}@{url.host}/{url.database}" @@ -74,6 +75,14 @@ def _add_entry( session.add(campaign) session.commit() # Commit to ensure instrument is saved and has an ID + site = session.query(Site).filter_by( + name=site_name).first() + if not site: + # Add the site with specific campaign + site = Site(name=site_name, campaign=campaign) + session.add(site) + session.commit() + observer_list = [] for obs_name in observer_names: observer = session.query(Observer).filter_by( @@ -88,7 +97,7 @@ def _add_entry( # Now that the instrument exists, create the entry, notice we only need the instrument object new_entry = data_cls( instrument=instrument, observers=observer_list, - campaign=campaign, **kwargs + site=site, **kwargs ) session.add(new_entry) session.commit() @@ -109,12 +118,11 @@ def populated_points(self, db): } self._add_entry( db.url, PointData, 'magnaprobe', ["TEST"], - 'Grand Mesa', **row + 'Grand Mesa', 'the_middle', **row ) @pytest.fixture(scope="class") def populated_layer(self, db): - # Fake data to implement row = { 'date': date(2020, 1, 28), @@ -129,7 +137,7 @@ def populated_layer(self, db): } self._add_entry( db.url, LayerData, 'fakeinstrument', ["TEST"], - 'Grand Mesa', **row + 'Grand Mesa', 'the_side', **row ) @pytest.fixture(scope="class") @@ -243,6 +251,10 @@ def test_all_site_names(self, clz): result = clz().all_site_names assert result == ['Grand Mesa'] + def test_all_site_ids(self, clz): + result = clz().all_site_ids + assert result == ['the_side'] + def test_all_dates(self, clz): result = clz().all_dates assert result == [date(2020, 1, 28)] From 264d85f25af4cb6f92d8d7c52a12ba305c47a83e Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Fri, 13 Sep 2024 10:41:25 -0600 Subject: [PATCH 15/16] simplifying some queries, add some todos --- snowexsql/api.py | 43 +++++++++++----------------------- snowexsql/tables/__init__.py | 2 +- snowexsql/tables/campaign.py | 2 ++ snowexsql/tables/layer_data.py | 2 +- snowexsql/tables/point_data.py | 2 +- snowexsql/tables/site.py | 5 ++-- snowexsql/tables/site_data.py | 5 ++-- tests/test_api.py | 2 +- tests/test_db.py | 6 ++--- 9 files changed, 29 insertions(+), 40 deletions(-) diff --git a/snowexsql/api.py b/snowexsql/api.py index f1c4873..688a32e 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -11,8 +11,7 @@ from snowexsql.conversions import query_to_geopandas, raster_to_rasterio from snowexsql.db import get_db from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ - Observer, Site -from snowexsql.tables.campaign import Campaign + Observer, Site, Campaign from snowexsql.tables.layer_data import LayerObservers from snowexsql.tables.point_data import PointObservers @@ -21,7 +20,6 @@ # TODO: # * Possible enums -# * filtering based on dates # * implement 'like' or 'contains' method @@ -194,14 +192,17 @@ def all_site_names(self): Return all campaign names """ with db_session(self.DB_NAME) as (session, engine): - qry = ( - session.query(Campaign.name) # Selecting Campaign names - .join(Site, - Site.campaign_id == Campaign.id) # Join Site to Campaign - .join(self.MODEL, - self.MODEL.site_id == Site.id) - .distinct() # To get distinct Campaign names - ) + qry = session.query(Campaign.name).distinct() + result = qry.all() + return self.retrieve_single_value_result(result) + + @property + def all_site_ids(self): + """ + Return all specific site names + """ + with db_session(self.DB_NAME) as (session, engine): + qry = session.query(Site.name).distinct() result = qry.all() return self.retrieve_single_value_result(result) @@ -231,13 +232,9 @@ def all_observers(self): Return all distinct observers in the data """ with db_session(self.DB_NAME) as (session, engine): - qry = session.query(Observer).join( - self.LINK_TABLE_MODEL, - Observer.id == self.LINK_TABLE_MODEL.observer_id - ).distinct() + qry = session.query(Observer.name).distinct() result = qry.all() - # Join the names - return [r.name for r in result] + return self.retrieve_single_value_result(result) @property def all_units(self): @@ -355,18 +352,6 @@ class LayerMeasurements(PointMeasurements): ] LINK_TABLE_MODEL = LayerObservers - @property - def all_site_ids(self): - """ - Return all types of the data - """ - with db_session(self.DB_NAME) as (session, engine): - qry = session.query(Site.name).join( - self.MODEL, Site.id == self.MODEL.site_id - ).distinct() - result = qry.all() - return self.retrieve_single_value_result(result) - class RasterMeasurements(BaseDataset): MODEL = ImageData diff --git a/snowexsql/tables/__init__.py b/snowexsql/tables/__init__.py index b188446..a6d3309 100644 --- a/snowexsql/tables/__init__.py +++ b/snowexsql/tables/__init__.py @@ -1,7 +1,7 @@ from .image_data import ImageData from .layer_data import LayerData from .point_data import PointData -from .site_data import SiteData +from .site_data import SiteCondition from .observers import Observer from .instrument import Instrument from .campaign import Campaign diff --git a/snowexsql/tables/campaign.py b/snowexsql/tables/campaign.py index f002e91..fed8608 100644 --- a/snowexsql/tables/campaign.py +++ b/snowexsql/tables/campaign.py @@ -17,6 +17,8 @@ class Campaign(Base): """ __tablename__ = 'campaigns' + # TODO: could we add a campaign shapefile? + id = Column(Integer, primary_key=True) name = Column(String()) description = Column(String()) diff --git a/snowexsql/tables/layer_data.py b/snowexsql/tables/layer_data.py index 6a4e0fd..497a1e6 100644 --- a/snowexsql/tables/layer_data.py +++ b/snowexsql/tables/layer_data.py @@ -50,7 +50,7 @@ class LayerData(SingleLocationData, Measurement, Base): # Link the site id with a foreign key site_id = Column( - Integer, ForeignKey('public.campaign_sites.id'), index=True + Integer, ForeignKey('public.sites.id'), index=True ) # Link the Site class site = relationship('Site') diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index 9607611..fcafd72 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -46,7 +46,7 @@ class PointData(SingleLocationData, Measurement, Base): # Link the site id with a foreign key site_id = Column( - Integer, ForeignKey('public.campaign_sites.id'), index=True + Integer, ForeignKey('public.sites.id'), index=True ) # Link the Site class site = relationship('Site') diff --git a/snowexsql/tables/site.py b/snowexsql/tables/site.py index 4dce286..e7db07a 100644 --- a/snowexsql/tables/site.py +++ b/snowexsql/tables/site.py @@ -5,7 +5,7 @@ @author: jtmaz """ -from sqlalchemy import Column, String, Integer, ForeignKey +from sqlalchemy import Column, String, Integer, ForeignKey, Float from sqlalchemy.orm import relationship from .base import Base @@ -17,7 +17,8 @@ class Site(Base): Table stores Site data. Does not store data values, it only stores the site metadata. """ - __tablename__ = 'campaign_sites' + # TODO: add geometry here and remove from site_condtions + __tablename__ = 'sites' id = Column(Integer, primary_key=True) name = Column(String()) diff --git a/snowexsql/tables/site_data.py b/snowexsql/tables/site_data.py index 5f1d1b0..1ea240e 100644 --- a/snowexsql/tables/site_data.py +++ b/snowexsql/tables/site_data.py @@ -3,12 +3,13 @@ from .base import Base, SingleLocationData -class SiteData(SingleLocationData, Base): +class SiteCondition(SingleLocationData, Base): """ Table for storing pit site meta data, This table doesn't represent any main data record but only support data for each site """ - __tablename__ = 'sites' + # TODO: leaving this for later - we should link this to Sites table + __tablename__ = 'site_condition' pit_id = Column(String(50)) slope_angle = Column(Float) diff --git a/tests/test_api.py b/tests/test_api.py index 68ea1d2..d2317c8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -253,7 +253,7 @@ def test_all_site_names(self, clz): def test_all_site_ids(self, clz): result = clz().all_site_ids - assert result == ['the_side'] + assert result == ['the_middle', 'the_side'] def test_all_dates(self, clz): result = clz().all_dates diff --git a/tests/test_db.py b/tests/test_db.py index c7624ba..f632453 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -4,7 +4,7 @@ from sqlalchemy import Table from snowexsql.db import get_db, get_table_attributes -from snowexsql.tables import ImageData, LayerData, PointData, SiteData +from snowexsql.tables import ImageData, LayerData, PointData, SiteCondition from .sql_test_base import DBSetup @@ -14,7 +14,7 @@ class TestDB(DBSetup): meas_atts = ['type', 'units'] - site_atts = base_atts + single_loc_atts + \ + site_atts = single_loc_atts + \ ['slope_angle', 'aspect', 'air_temp', 'total_depth', 'weather_description', 'precip', 'sky_cover', 'wind', 'ground_condition', 'ground_roughness', @@ -59,7 +59,7 @@ def test_layer_structure(self): assert c in columns @pytest.mark.parametrize("DataCls,attributes", [ - (SiteData, site_atts), + (SiteCondition, site_atts), (PointData, point_atts), (LayerData, layer_atts), (ImageData, raster_atts)]) From c232a17c18dc2d5c0e2da93b96e3615a8e1b9ead Mon Sep 17 00:00:00 2001 From: Micah Sandusky Date: Mon, 16 Sep 2024 10:28:47 -0600 Subject: [PATCH 16/16] link table unused imports --- snowexsql/api.py | 5 +---- snowexsql/tables/layer_data.py | 1 - snowexsql/tables/point_data.py | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/snowexsql/api.py b/snowexsql/api.py index 688a32e..acc2e6d 100644 --- a/snowexsql/api.py +++ b/snowexsql/api.py @@ -12,8 +12,7 @@ from snowexsql.db import get_db from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \ Observer, Site, Campaign -from snowexsql.tables.layer_data import LayerObservers -from snowexsql.tables.point_data import PointObservers + LOG = logging.getLogger(__name__) DB_NAME = 'snow:hackweek@db.snowexdata.org/snowex' @@ -46,7 +45,6 @@ def get_points(): class BaseDataset: MODEL = None - LINK_TABLE_MODEL = PointObservers # Use this database name DB_NAME = DB_NAME @@ -350,7 +348,6 @@ class LayerMeasurements(PointMeasurements): "campaign", "site_id", "date", "instrument", "observer", "type", "utm_zone", "pit_id", "date_greater_equal", "date_less_equal" ] - LINK_TABLE_MODEL = LayerObservers class RasterMeasurements(BaseDataset): diff --git a/snowexsql/tables/layer_data.py b/snowexsql/tables/layer_data.py index 497a1e6..e7fb93f 100644 --- a/snowexsql/tables/layer_data.py +++ b/snowexsql/tables/layer_data.py @@ -17,7 +17,6 @@ class LayerObservers(Base): __tablename__ = 'layer_observers' __table_args__ = {'schema': 'public'} - id = Column(Integer, primary_key=True) layer_id = Column(Integer, ForeignKey('public.layers.id')) observer_id = Column(Integer, ForeignKey("public.observers.id")) diff --git a/snowexsql/tables/point_data.py b/snowexsql/tables/point_data.py index fcafd72..ceb691c 100644 --- a/snowexsql/tables/point_data.py +++ b/snowexsql/tables/point_data.py @@ -16,7 +16,6 @@ class PointObservers(Base): """ __tablename__ = 'point_observers' - id = Column(Integer, primary_key=True) point_id = Column(Integer, ForeignKey('public.points.id')) observer_id = Column(Integer, ForeignKey("public.observers.id"))