Merge dev work from hackweek and last few weeks into a target release branch

snowexsql/api.py

@@ -10,7 +10,11 @@
 
 from snowexsql.conversions import query_to_geopandas, raster_to_rasterio
 from snowexsql.db import get_db
-from snowexsql.tables import ImageData, LayerData, PointData
+from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \
+    Observer
+from snowexsql.tables.campaign import Campaign
+from snowexsql.tables.layer_data import LayerObservers
+from snowexsql.tables.point_data import PointObservers
 
 LOG = logging.getLogger(__name__)
 DB_NAME = 'snow:[email protected]/snowex'
@@ -44,11 +48,14 @@ def get_points():
 
 class BaseDataset:
     MODEL = None
+    LINK_TABLE_MODEL = PointObservers
     # Use this database name
     DB_NAME = DB_NAME
 
-    ALLOWED_QRY_KWARGS = ["site_name", "site_id", "date", "instrument", "observers", "type",
-        "utm_zone", "date_greater_equal", "date_less_equal", "value_greater_equal", 'value_less_equal',
+    ALLOWED_QRY_KWARGS = [
+        "campaign", "site_id", "date", "instrument", "type",
+        "utm_zone", "date_greater_equal", "date_less_equal",
+        "value_greater_equal", 'value_less_equal',
     ]
     SPECIAL_KWARGS = ["limit"]
     # Default max record count
@@ -119,6 +126,19 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
                         key = k.split("_less_equal")[0]
                         filter_col = getattr(cls.MODEL, key)
                         qry = qry.filter(filter_col <= v)
+                    # Filter linked columns
+                    elif k == "instrument":
+                        qry = qry.filter(
+                            cls.MODEL.instrument.has(name=v)
+                        )
+                    elif k == "campaign":
+                        qry = qry.filter(
+                            cls.MODEL.campaign.has(name=v)
+                        )
+                    elif k == "observer":
+                        qry = qry.join(
+                            LayerData.observers
+                        ).filter(Observer.name == v)
                     # Filter to exact value
                     else:
                         filter_col = getattr(cls.MODEL, k)
@@ -168,7 +188,9 @@ def all_site_names(self):
         Return all types of the data
         """
         with db_session(self.DB_NAME) as (session, engine):
-            qry = session.query(self.MODEL.site_name).distinct()
+            qry = session.query(Campaign.name).join(
+                self.MODEL, Campaign.id == self.MODEL.campaign_id
+            ).distinct()
             result = qry.all()
         return self.retrieve_single_value_result(result)
 
@@ -198,9 +220,13 @@ def all_observers(self):
         Return all distinct observers in the data
         """
         with db_session(self.DB_NAME) as (session, engine):
-            qry = session.query(self.MODEL.observers).distinct()
+            qry = session.query(Observer).join(
+                self.LINK_TABLE_MODEL,
+                Observer.id == self.LINK_TABLE_MODEL.observer_id
+            ).distinct()
             result = qry.all()
-        return self.retrieve_single_value_result(result)
+        # Join the names
+        return [r.name for r in result]
 
     @property
     def all_units(self):
@@ -218,7 +244,9 @@ def all_instruments(self):
         Return all distinct instruments in the data
         """
         with db_session(self.DB_NAME) as (session, engine):
-            qry = session.query(self.MODEL.instrument).distinct()
+            qry = session.query(Instrument.name).join(
+                self.MODEL, Instrument.id == self.MODEL.instrument_id
+            ).distinct()
             result = qry.all()
         return self.retrieve_single_value_result(result)
 
@@ -299,20 +327,22 @@ def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs):
 
         return df
 
+
 class TooManyRastersException(Exception):
     """ Exceptiont to report to users that their query will produce too many rasters"""
     pass
 
+
 class LayerMeasurements(PointMeasurements):
     """
     API class for access to LayerData
     """
     MODEL = LayerData
     ALLOWED_QRY_KWARGS = [
-        "site_name", "site_id", "date", "instrument", "observers", "type",
+        "campaign", "site_id", "date", "instrument", "observer", "type",
         "utm_zone", "pit_id", "date_greater_equal", "date_less_equal"
     ]
-    # TODO: layer analysis methods?
+    LINK_TABLE_MODEL = LayerObservers
 
     @property
     def all_site_ids(self):

snowexsql/tables/__init__.py

@@ -2,10 +2,15 @@
 from .layer_data import LayerData
 from .point_data import PointData
 from .site_data import SiteData
+from .observers import Observer
+from .instrument import Instrument
+from .campaign import Campaign
 
 __all__ = [
-    'ImageData',
-    'LayerData',
-    'PointData',
-    'SnowData',
+    "Campaign",
+    "ImageData",
+    "Instrument",
+    "LayerData",
+    "Observer",
+    "PointData",  
 ]

snowexsql/tables/base.py

@@ -26,7 +26,6 @@ class Base(DeclarativeBase):
     time_updated = Column(DateTime(timezone=True), onupdate=func.now())
 
     date_accessed = Column(Date)
-    site_name = Column(String(250))
     date = Column(Date)
     doi = Column(String(50))
 
@@ -45,7 +44,5 @@ class Measurement(object):
     """
     Base Class providing attributes required for a measurement of any kind
     """
-    instrument = Column(String(50))
     type = Column(String(50))
     units = Column(String(50))
-    observers = Column(String(100))

snowexsql/tables/campaign.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 22 11:56:34 2024
+
+@author: jtmaz
+"""
+
+from sqlalchemy import Column, String, Integer
+
+from .base import Base 
+
+
+class Campaign(Base):
+    """
+    Table stores Campaign data. Does not store data values,
+    it only stores the campaign metadata. 
+    """
+    __tablename__ = 'campaigns'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String())
+    description = Column(String())

snowexsql/tables/instrument.py

@@ -0,0 +1,12 @@
+from sqlalchemy import Column, Integer, String
+from .base import Base
+
+
+class Instrument(Base):
+    __tablename__ = 'instruments'
+    # auto created id
+    id = Column(Integer, primary_key=True)
+    # Name of the instrument
+    name = Column(String(), index=True)
+    model = Column(String())
+    specifications = Column(String())

snowexsql/tables/layer_data.py

@@ -1,6 +1,25 @@
-from sqlalchemy import Column, Float, String
+from sqlalchemy import Column, Float, Integer, String, ForeignKey
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+from typing import List
+from sqlalchemy.orm import relationship
 
 from .base import Base, Measurement, SingleLocationData
+from .observers import Observer
+from .instrument import Instrument
+from .campaign import Campaign
+
+
+class LayerObservers(Base):
+    """
+    Link table
+    """
+    __tablename__ = 'layer_observers'
+    __table_args__ = {'schema': 'public'}
+
+    id = Column(Integer, primary_key=True)
+    layer_id = Column(Integer, ForeignKey('public.layers.id'))
+    observer_id = Column(Integer, ForeignKey("public.observers.id"))
 
 
 class LayerData(SingleLocationData, Measurement, Base):
@@ -22,3 +41,23 @@ class LayerData(SingleLocationData, Measurement, Base):
     sample_c = Column(String(20))
     value = Column(String(50))
     flags = Column(String(20))
+
+    # Link the instrument id with a foreign key
+    instrument_id = Column(
+        Integer, ForeignKey('public.instruments.id'), index=True
+    )
+    # Link the Instrument class
+    instrument = relationship('Instrument')
+
+    # Link the campaign id with a foreign key
+    campaign_id = Column(
+        Integer, ForeignKey('public.campaigns.id'), index=True
+    )
+    # Link the Campaign class
+    campaign = relationship('Campaign')
+
+    # id is a mapped column for many-to-many with observers
+    id: Mapped[int] = mapped_column(primary_key=True)
+    observers: Mapped[List[Observer]] = relationship(
+        secondary=LayerObservers.__table__
+    )

snowexsql/tables/observers.py

@@ -0,0 +1,12 @@
+from sqlalchemy.orm import mapped_column, Mapped
+from sqlalchemy import Column, String
+
+from .base import Base
+
+
+class Observer(Base):
+    __tablename__ = 'observers'
+    # id is mapped column for many-to-many
+    id: Mapped[int] = mapped_column(primary_key=True)
+    # Name of the observer
+    name = Column(String())

snowexsql/tables/point_data.py

@@ -1,6 +1,24 @@
-from sqlalchemy import Column, Float, Integer, String
+from sqlalchemy import Column, Float, Integer, String, ForeignKey
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+from typing import List
+from sqlalchemy.orm import relationship
 
 from .base import Base, Measurement, SingleLocationData
+from .observers import Observer
+from .instrument import Instrument
+from .campaign import Campaign
+
+
+class PointObservers(Base):
+    """
+    Link table
+    """
+    __tablename__ = 'point_observers'
+
+    id = Column(Integer, primary_key=True)
+    point_id = Column(Integer, ForeignKey('public.points.id'))
+    observer_id = Column(Integer, ForeignKey("public.observers.id"))
 
 
 class PointData(SingleLocationData, Measurement, Base):
@@ -12,5 +30,29 @@ class PointData(SingleLocationData, Measurement, Base):
     __tablename__ = 'points'
 
     version_number = Column(Integer)
-    equipment = Column(String(50))
+    equipment = Column(String())
     value = Column(Float)
+
+    # bring these in instead of Measurement
+    type = Column(String())
+    units = Column(String())
+
+    # Link the instrument id with a foreign key
+    instrument_id = Column(
+        Integer, ForeignKey('public.instruments.id'), index=True
+    )
+    # Link the Instrument class
+    instrument = relationship('Instrument')
+
+    # Link the campaign id with a foreign key
+    campaign_id = Column(
+        Integer, ForeignKey('public.campaigns.id'), index=True
+    )
+    # Link the Campaign class
+    campaign = relationship('Campaign')
+
+    # id is a mapped column for many-to-many with observers
+    id: Mapped[int] = mapped_column(primary_key=True)
+    observers: Mapped[List[Observer]] = relationship(
+        secondary=PointObservers.__table__
+    )

snowexsql/tables/site_data.py

@@ -9,7 +9,6 @@ class SiteData(SingleLocationData, Base):
     main data record but only support data for each site
     """
     __tablename__ = 'sites'
-    __table_args__ = {"schema": "public"}
 
     pit_id = Column(String(50))
     slope_angle = Column(Float)

tests/sql_test_base.py

@@ -42,9 +42,8 @@ def setup_class(self):
     @classmethod
     def teardown_class(self):
         """
-        Remove the databse
+        Close the session
         """
-        self.metadata.drop_all(bind=self.engine)
         self.session.close()  # optional, depends on use case
 
     def teardown(self):