From 328563970ea49a4ea786774a55d6923fafdbeaf6 Mon Sep 17 00:00:00 2001 From: Shaibu Shaibu Date: Sun, 5 Sep 2021 00:41:39 +0100 Subject: [PATCH 1/7] update requirements file and generate model definition for 24 tables using sqlacodegen --- .gitignore | 1 + opencdms/models/climsoft.py | 382 ++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 384 insertions(+) create mode 100644 opencdms/models/climsoft.py diff --git a/.gitignore b/.gitignore index 4c915d14..7a4a5233 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ ENV/ # IDE settings .vscode/ .idea/ +shaibu-activity.log diff --git a/opencdms/models/climsoft.py b/opencdms/models/climsoft.py new file mode 100644 index 00000000..cd1479f2 --- /dev/null +++ b/opencdms/models/climsoft.py @@ -0,0 +1,382 @@ +# coding: utf-8 +from sqlalchemy import BigInteger, CHAR, Column, DECIMAL, DateTime, Float, ForeignKey, Index, Integer, String, Text, text +from sqlalchemy.dialects.mysql import TINYINT +from sqlalchemy.orm import relationship +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() +metadata = Base.metadata + + +class Acquisitiontype(Base): + __tablename__ = 'acquisitiontype' + + code = Column(Integer, primary_key=True, server_default=text("'0'")) + description = Column(String(255)) + + +class DataForm(Base): + __tablename__ = 'data_forms' + + id = Column(BigInteger, nullable=False, server_default=text("'0'")) + order_num = Column(BigInteger, server_default=text("'0'")) + table_name = Column(String(255)) + form_name = Column(String(250), primary_key=True) + description = Column(Text) + selected = Column(TINYINT) + val_start_position = Column(BigInteger, server_default=text("'0'")) + val_end_position = Column(BigInteger, server_default=text("'0'")) + elem_code_location = Column(String(255)) + sequencer = Column(String(50)) + entry_mode = Column(TINYINT(2), nullable=False, server_default=text("'00'")) + + +class Flag(Base): + __tablename__ = 'flags' + + characterSymbol = Column(String(255), primary_key=True, server_default=text("''")) + numSymbol = Column(Integer) + description = Column(String(255)) + + +class Obselement(Base): + __tablename__ = 'obselement' + + elementId = Column(BigInteger, primary_key=True, index=True, server_default=text("'0'")) + abbreviation = Column(String(255)) + elementName = Column(String(255)) + description = Column(String(255)) + elementScale = Column(DECIMAL(8, 2)) + upperLimit = Column(String(255)) + lowerLimit = Column(String(255)) + units = Column(String(255)) + elementtype = Column(String(50)) + qcTotalRequired = Column(Integer, server_default=text("'0'")) + selected = Column(TINYINT, nullable=False, server_default=text("'0'")) + + +class Paperarchivedefinition(Base): + __tablename__ = 'paperarchivedefinition' + + formId = Column(String(50), primary_key=True, index=True) + description = Column(String(255)) + + +class Qcstatusdefinition(Base): + __tablename__ = 'qcstatusdefinition' + + code = Column(Integer, primary_key=True, server_default=text("'0'")) + description = Column(String(255)) + + +class Qctype(Base): + __tablename__ = 'qctype' + + code = Column(Integer, primary_key=True, server_default=text("'0'")) + description = Column(String(255)) + + +class Regkey(Base): + __tablename__ = 'regkeys' + + keyName = Column(String(255), primary_key=True, server_default=text("''")) + keyValue = Column(String(255)) + keyDescription = Column(String(255)) + + +class Station(Base): + __tablename__ = 'station' + + stationId = Column(String(255), primary_key=True, index=True) + stationName = Column(String(255)) + wmoid = Column(String(20)) + icaoid = Column(String(20)) + latitude = Column(Float(11, True)) + qualifier = Column(String(20)) + longitude = Column(Float(11, True)) + elevation = Column(String(255)) + geoLocationMethod = Column(String(255)) + geoLocationAccuracy = Column(Float(11)) + openingDatetime = Column(String(50)) + closingDatetime = Column(String(50)) + country = Column(String(50)) + authority = Column(String(255)) + adminRegion = Column(String(255)) + drainageBasin = Column(String(255)) + wacaSelection = Column(TINYINT, server_default=text("'0'")) + cptSelection = Column(TINYINT, server_default=text("'0'")) + stationOperational = Column(TINYINT, server_default=text("'0'")) + + +class Synopfeature(Base): + __tablename__ = 'synopfeature' + + abbreviation = Column(String(255), primary_key=True) + description = Column(String(255)) + + +class Featuregeographicalposition(Base): + __tablename__ = 'featuregeographicalposition' + __table_args__ = ( + Index('FK_mysql_climsoft_db_v4_synopfeatureFeatureGeographicalPosition', 'belongsTo', 'observedOn', unique=True), + ) + + belongsTo = Column(ForeignKey('synopfeature.abbreviation'), primary_key=True, nullable=False) + observedOn = Column(String(50), primary_key=True, nullable=False) + latitude = Column(Float(11, True)) + longitude = Column(Float(11, True)) + + synopfeature = relationship('Synopfeature') + + +class Instrument(Base): + __tablename__ = 'instrument' + + instrumentName = Column(String(255)) + instrumentId = Column(String(255), primary_key=True, index=True) + serialNumber = Column(String(255)) + abbreviation = Column(String(255)) + model = Column(String(255)) + manufacturer = Column(String(255)) + instrumentUncertainty = Column(Float(11)) + installationDatetime = Column(String(50)) + deinstallationDatetime = Column(String(50)) + height = Column(String(255)) + instrumentPicture = Column(CHAR(255)) + installedAt = Column(ForeignKey('station.stationId'), index=True) + + station = relationship('Station') + + +class Observationfinal(Base): + __tablename__ = 'observationfinal' + __table_args__ = ( + Index('obsFinalIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', unique=True), + ) + + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + obsDatetime = Column(DateTime, primary_key=True, nullable=False) + obsLevel = Column(String(255), primary_key=True, nullable=False, server_default=text("'surface'")) + obsValue = Column(DECIMAL(8, 2)) + flag = Column(String(255), server_default=text("'N'")) + period = Column(Integer) + qcStatus = Column(Integer, server_default=text("'0'")) + qcTypeLog = Column(Text) + acquisitionType = Column(Integer, server_default=text("'0'")) + dataForm = Column(String(255)) + capturedBy = Column(String(255)) + mark = Column(TINYINT) + temperatureUnits = Column(String(255)) + precipitationUnits = Column(String(255)) + cloudHeightUnits = Column(String(255)) + visUnits = Column(String(255)) + dataSourceTimeZone = Column(Integer, server_default=text("'0'")) + + obselement = relationship('Obselement') + station = relationship('Station') + + +class Observationinitial(Base): + __tablename__ = 'observationinitial' + __table_args__ = ( + Index('obsInitialIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', 'qcStatus', 'acquisitionType', unique=True), + ) + + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + obsDatetime = Column(DateTime, primary_key=True, nullable=False) + obsLevel = Column(String(255), primary_key=True, nullable=False) + obsValue = Column(String(255)) + flag = Column(String(255)) + period = Column(Integer) + qcStatus = Column(Integer, primary_key=True, nullable=False, server_default=text("'0'")) + qcTypeLog = Column(Text) + acquisitionType = Column(Integer, primary_key=True, nullable=False, server_default=text("'0'")) + dataForm = Column(String(255)) + capturedBy = Column(String(255)) + mark = Column(TINYINT) + temperatureUnits = Column(String(255)) + precipitationUnits = Column(String(255)) + cloudHeightUnits = Column(String(255)) + visUnits = Column(String(255)) + dataSourceTimeZone = Column(Integer, server_default=text("'0'")) + + obselement = relationship('Obselement') + station = relationship('Station') + + +class Obsscheduleclas(Base): + __tablename__ = 'obsscheduleclass' + + scheduleClass = Column(String(255), primary_key=True, index=True, server_default=text("''")) + description = Column(String(255)) + refersTo = Column(ForeignKey('station.stationId'), index=True) + + station = relationship('Station') + + +class Paperarchive(Base): + __tablename__ = 'paperarchive' + __table_args__ = ( + Index('paper_archive_identification', 'belongsTo', 'formDatetime', 'classifiedInto', unique=True), + ) + + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + formDatetime = Column(DateTime, primary_key=True, nullable=False) + image = Column(String(255)) + classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False, index=True) + + station = relationship('Station') + paperarchivedefinition = relationship('Paperarchivedefinition') + + +class Physicalfeatureclas(Base): + __tablename__ = 'physicalfeatureclass' + + featureClass = Column(String(255), primary_key=True, index=True) + description = Column(String(255)) + refersTo = Column(ForeignKey('station.stationId'), index=True) + + station = relationship('Station') + + +class Stationlocationhistory(Base): + __tablename__ = 'stationlocationhistory' + __table_args__ = ( + Index('history', 'belongsTo', 'openingDatetime', unique=True), + ) + + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + stationType = Column(String(255)) + geoLocationMethod = Column(String(255)) + geoLocationAccuracy = Column(Float(11)) + openingDatetime = Column(String(50), primary_key=True, nullable=False) + closingDatetime = Column(String(50)) + latitude = Column(Float(11, True)) + longitude = Column(Float(11, True)) + elevation = Column(BigInteger) + authority = Column(String(255)) + adminRegion = Column(String(255)) + drainageBasin = Column(String(255)) + + station = relationship('Station') + + +class Stationqualifier(Base): + __tablename__ = 'stationqualifier' + __table_args__ = ( + Index('stationid_qualifier_identification', 'qualifier', 'qualifierBeginDate', 'qualifierEndDate', 'belongsTo', unique=True), + ) + + qualifier = Column(String(255), primary_key=True, nullable=False) + qualifierBeginDate = Column(String(50), primary_key=True, nullable=False) + qualifierEndDate = Column(String(50), primary_key=True, nullable=False) + stationTimeZone = Column(Integer, server_default=text("'0'")) + stationNetworkType = Column(String(255)) + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + + station = relationship('Station') + + +class Instrumentfaultreport(Base): + __tablename__ = 'instrumentfaultreport' + __table_args__ = ( + Index('instrument_report', 'refersTo', 'reportDatetime', 'reportedFrom', unique=True), + ) + + refersTo = Column(ForeignKey('instrument.instrumentId')) + reportId = Column(BigInteger, primary_key=True, index=True) + reportDatetime = Column(String(50)) + faultDescription = Column(String(255)) + reportedBy = Column(String(255)) + receivedDatetime = Column(String(50)) + receivedBy = Column(String(255)) + reportedFrom = Column(ForeignKey('station.stationId'), index=True) + + instrument = relationship('Instrument') + station = relationship('Station') + + +class Instrumentinspection(Base): + __tablename__ = 'instrumentinspection' + __table_args__ = ( + Index('inspection', 'performedOn', 'inspectionDatetime', unique=True), + ) + + performedOn = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False) + inspectionDatetime = Column(String(50), primary_key=True, nullable=False) + performedBy = Column(String(255)) + status = Column(String(255)) + remarks = Column(String(255)) + performedAt = Column(ForeignKey('station.stationId'), index=True) + + station = relationship('Station') + instrument = relationship('Instrument') + + +class Observationschedule(Base): + __tablename__ = 'observationschedule' + __table_args__ = ( + Index('scheduleIdentification', 'classifiedInto', 'startTime', 'endTime', unique=True), + ) + + classifiedInto = Column(ForeignKey('obsscheduleclass.scheduleClass'), primary_key=True, nullable=False) + startTime = Column(String(50), primary_key=True, nullable=False) + endTime = Column(String(50), primary_key=True, nullable=False) + interval = Column(String(255)) + additionalObsTime = Column(String(255)) + + obsscheduleclas = relationship('Obsscheduleclas') + + +class Physicalfeature(Base): + __tablename__ = 'physicalfeature' + __table_args__ = ( + Index('featureIdentification', 'associatedWith', 'beginDate', 'classifiedInto', 'description', unique=True), + ) + + associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + beginDate = Column(String(50), primary_key=True, nullable=False) + endDate = Column(String(50)) + image = Column(String(255)) + description = Column(String(255), primary_key=True, nullable=False) + classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False, index=True) + + station = relationship('Station') + physicalfeatureclas = relationship('Physicalfeatureclas') + + +class Stationelement(Base): + __tablename__ = 'stationelement' + __table_args__ = ( + Index('stationElementIdentification', 'recordedFrom', 'describedBy', 'recordedWith', 'beginDate', unique=True), + ) + + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False, index=True) + instrumentcode = Column(String(6)) + scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass'), index=True) + height = Column(Float(6)) + beginDate = Column(String(50), primary_key=True, nullable=False) + endDate = Column(String(50)) + + obselement = relationship('Obselement') + station = relationship('Station') + instrument = relationship('Instrument') + obsscheduleclas = relationship('Obsscheduleclas') + + +class Faultresolution(Base): + __tablename__ = 'faultresolution' + __table_args__ = ( + Index('solution', 'resolvedDatetime', 'associatedWith', unique=True), + ) + + resolvedDatetime = Column(String(50), primary_key=True, nullable=False) + resolvedBy = Column(String(255)) + associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False, index=True) + remarks = Column(String(255)) + + instrumentfaultreport = relationship('Instrumentfaultreport') diff --git a/requirements.txt b/requirements.txt index 91c51f5f..028cc477 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pandas==1.3.1 psycopg2==2.9.1 sqlacodegen==2.3.0 SQLAlchemy==1.4.22 +mysqlclient==2.0.3 From dc187db4cfd0a86392cab8d150df55afb1042151 Mon Sep 17 00:00:00 2001 From: Shaibu Shaibu Date: Mon, 6 Sep 2021 21:42:40 +0100 Subject: [PATCH 2/7] Support index operations with get_schema_diff function and move climsoft model files to v4_1_1_core.py --- opencdms/models/__init__.py | 71 ++++++++++++++----- opencdms/models/climsoft/__init__.py | 0 .../{climsoft.py => climsoft/v4_1_1_core.py} | 28 ++++++++ 3 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 opencdms/models/climsoft/__init__.py rename opencdms/models/{climsoft.py => climsoft/v4_1_1_core.py} (96%) diff --git a/opencdms/models/__init__.py b/opencdms/models/__init__.py index 855b662b..485db632 100644 --- a/opencdms/models/__init__.py +++ b/opencdms/models/__init__.py @@ -1,10 +1,11 @@ +from pprint import pprint from typing import Callable, List, Optional, Sequence from alembic import migration -from sqlalchemy import MetaData, create_engine +from sqlalchemy import MetaData, create_engine, Table, Index from alembic.autogenerate import compare_metadata -def filter_ignored_tables( +def filter_out_excluded_tables( tables: Sequence[str], ) -> Callable[[tuple], bool]: """ Accept sequence of table names to be ignored when comparing model to\ @@ -23,10 +24,40 @@ def filter_func(item: tuple): """ Return true if item's table name not in `tables` """ - operation: str = item[0] - if operation == "add_table": - _, table = item - return table.name not in tables + target_object = item[1] + if isinstance(target_object, Table): + return target_object.name not in tables + if isinstance(target_object, Index): + return target_object.table.name not in tables + return True + + return filter_func + + +def filter_in_included_tables( + tables: Sequence[str], +) -> Callable[[tuple], bool]: + """ Accept sequence of table names to be included when comparing model to\ + db schema + Parameters + ----- + `tables`: Sequence[str] + Table names to be included, other tables not in this list would be ignored. + + Return + ------ + `Callable[[tuple]], bool]` + """ + + def filter_func(item: tuple): + """ + Return true if item's table name not in `tables` + """ + target_object = item[1] + if isinstance(target_object, Table): + return target_object.name in tables + if isinstance(target_object, Index): + return target_object.table.name in tables return True return filter_func @@ -35,7 +66,8 @@ def filter_func(item: tuple): def get_schema_diff( metadata: MetaData, database_url: str, - ignore_tables: Optional[Sequence[str]] = None, + include_tables: Optional[Sequence[str]] = None, + exclude_tables: Optional[Sequence[str]] = None, ) -> List[tuple]: """ Parameters @@ -44,7 +76,9 @@ def get_schema_diff( Sqlalchemy Metadata `database_url`: Target databse url e.g. `sqlite://` - `ignore_tables`: Sequence[str] | None + `include_tables`: Sequence[str] | None + List of table names to check against the database + `exclude_tables`: Sequence[str] | None List of table names to be ignored Return @@ -55,19 +89,22 @@ def get_schema_diff( engine = create_engine(database_url) mc = migration.MigrationContext.configure(engine.connect()) diff = compare_metadata(mc, metadata) - if ignore_tables is not None: - diff = list(filter(filter_ignored_tables(ignore_tables), diff)) + if include_tables is not None and exclude_tables is not None: + raise Exception("`include_tables` and `exclude_tables` must not be used together") #TODO define custom error class + if exclude_tables is not None: + diff = list(filter(filter_out_excluded_tables(exclude_tables), diff)) + if include_tables is not None: + diff = list(filter(filter_in_included_tables(include_tables), diff)) return list(diff) # TODO remove and use automated tests instead. if __name__ == "__main__": - from opencdms.models.clide import metadata, CLIDE_VIEWS - - print( - get_schema_diff( + from opencdms.models.climsoft.v4_1_1_core import metadata, TARGET_TABLES + diff = get_schema_diff( metadata, - "postgresql+psycopg2://localhost/clideDB", - ignore_tables=CLIDE_VIEWS, + "mysql://root:#Classicnerdswag1@localhost/mariadb_climsoft_db_v4", + include_tables=TARGET_TABLES ) - ) + with open('output.txt', 'wt') as out: + pprint(diff, stream=out, indent=4) diff --git a/opencdms/models/climsoft/__init__.py b/opencdms/models/climsoft/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/opencdms/models/climsoft.py b/opencdms/models/climsoft/v4_1_1_core.py similarity index 96% rename from opencdms/models/climsoft.py rename to opencdms/models/climsoft/v4_1_1_core.py index cd1479f2..f2db0abe 100644 --- a/opencdms/models/climsoft.py +++ b/opencdms/models/climsoft/v4_1_1_core.py @@ -8,6 +8,34 @@ metadata = Base.metadata +TARGET_TABLES = [ + "acquisitiontype", + "data_forms", + "flags", + "obselement", + "paperarchivedefinition", + "qcstatusdefinition", + "qctype", + "regkeys", + "station", + "synopfeature", + "featuregeographicalposition", + "instrument", + "observationfinal", + "observationinitial", + "obsscheduleclass", + "paperarchive", + "physicalfeatureclass", + "stationlocationhistory", + "stationqualifier", + "instrumentfaultreport", + "instrumentinspection", + "observationschedule", + "physicalfeature", + "stationelement", + "faultresolution" +] + class Acquisitiontype(Base): __tablename__ = 'acquisitiontype' From aa075257ad9cc59e395d0d788495da689c415ce7 Mon Sep 17 00:00:00 2001 From: Shaibu Shaibu Date: Tue, 7 Sep 2021 00:15:47 +0100 Subject: [PATCH 3/7] update climsoft core models to match database --- opencdms/models/climsoft/v4_1_1_core.py | 66 +++++++++++++++---------- requirements.txt | 1 + 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/opencdms/models/climsoft/v4_1_1_core.py b/opencdms/models/climsoft/v4_1_1_core.py index f2db0abe..9e84a75a 100644 --- a/opencdms/models/climsoft/v4_1_1_core.py +++ b/opencdms/models/climsoft/v4_1_1_core.py @@ -69,8 +69,9 @@ class Flag(Base): class Obselement(Base): __tablename__ = 'obselement' + __table_args__ = ( Index('elementCode', 'elementId'),) - elementId = Column(BigInteger, primary_key=True, index=True, server_default=text("'0'")) + elementId = Column(BigInteger, primary_key=True, server_default=text("'0'")) abbreviation = Column(String(255)) elementName = Column(String(255)) description = Column(String(255)) @@ -85,8 +86,9 @@ class Obselement(Base): class Paperarchivedefinition(Base): __tablename__ = 'paperarchivedefinition' + __table_args__ = (Index('paperarchivedef','formId'),) - formId = Column(String(50), primary_key=True, index=True) + formId = Column(String(50), primary_key=True) description = Column(String(255)) @@ -114,8 +116,9 @@ class Regkey(Base): class Station(Base): __tablename__ = 'station' + __table_args__ = (Index('StationStationId', 'stationId'),) - stationId = Column(String(255), primary_key=True, index=True) + stationId = Column(String(255), primary_key=True) stationName = Column(String(255)) wmoid = Column(String(20)) icaoid = Column(String(20)) @@ -159,9 +162,10 @@ class Featuregeographicalposition(Base): class Instrument(Base): __tablename__ = 'instrument' + __table_args__ = (Index('code', 'instrumentId'),) instrumentName = Column(String(255)) - instrumentId = Column(String(255), primary_key=True, index=True) + instrumentId = Column(String(255), primary_key=True) serialNumber = Column(String(255)) abbreviation = Column(String(255)) model = Column(String(255)) @@ -171,7 +175,7 @@ class Instrument(Base): deinstallationDatetime = Column(String(50)) height = Column(String(255)) instrumentPicture = Column(CHAR(255)) - installedAt = Column(ForeignKey('station.stationId'), index=True) + installedAt = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -180,10 +184,12 @@ class Observationfinal(Base): __tablename__ = 'observationfinal' __table_args__ = ( Index('obsFinalIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', unique=True), + Index('obsElementObservationInitial', 'describedBy'), + Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False, server_default=text("'surface'")) obsValue = Column(DECIMAL(8, 2)) @@ -209,10 +215,12 @@ class Observationinitial(Base): __tablename__ = 'observationinitial' __table_args__ = ( Index('obsInitialIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', 'qcStatus', 'acquisitionType', unique=True), + Index('obsElementObservationInitial', 'describedBy'), + Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False) obsValue = Column(String(255)) @@ -236,10 +244,11 @@ class Observationinitial(Base): class Obsscheduleclas(Base): __tablename__ = 'obsscheduleclass' + __table_args__ = (Index('scheduleClassIdeification', 'scheduleClass'),) - scheduleClass = Column(String(255), primary_key=True, index=True, server_default=text("''")) + scheduleClass = Column(String(255), primary_key=True, server_default=text("''")) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId'), index=True) + refersTo = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -253,7 +262,7 @@ class Paperarchive(Base): belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) formDatetime = Column(DateTime, primary_key=True, nullable=False) image = Column(String(255)) - classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False, index=True) + classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False) station = relationship('Station') paperarchivedefinition = relationship('Paperarchivedefinition') @@ -261,10 +270,11 @@ class Paperarchive(Base): class Physicalfeatureclas(Base): __tablename__ = 'physicalfeatureclass' + __table_args__ = (Index('stationFeatureClass', 'featureClass'),) - featureClass = Column(String(255), primary_key=True, index=True) + featureClass = Column(String(255), primary_key=True) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId'), index=True) + refersTo = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -295,6 +305,7 @@ class Stationqualifier(Base): __tablename__ = 'stationqualifier' __table_args__ = ( Index('stationid_qualifier_identification', 'qualifier', 'qualifierBeginDate', 'qualifierEndDate', 'belongsTo', unique=True), + Index('stationQualifierIdentification', 'belongsTo') ) qualifier = Column(String(255), primary_key=True, nullable=False) @@ -302,7 +313,7 @@ class Stationqualifier(Base): qualifierEndDate = Column(String(50), primary_key=True, nullable=False) stationTimeZone = Column(Integer, server_default=text("'0'")) stationNetworkType = Column(String(255)) - belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) station = relationship('Station') @@ -311,16 +322,17 @@ class Instrumentfaultreport(Base): __tablename__ = 'instrumentfaultreport' __table_args__ = ( Index('instrument_report', 'refersTo', 'reportDatetime', 'reportedFrom', unique=True), + Index('report_id', 'reportId') ) refersTo = Column(ForeignKey('instrument.instrumentId')) - reportId = Column(BigInteger, primary_key=True, index=True) + reportId = Column(BigInteger, primary_key=True) reportDatetime = Column(String(50)) faultDescription = Column(String(255)) reportedBy = Column(String(255)) receivedDatetime = Column(String(50)) receivedBy = Column(String(255)) - reportedFrom = Column(ForeignKey('station.stationId'), index=True) + reportedFrom = Column(ForeignKey('station.stationId')) instrument = relationship('Instrument') station = relationship('Station') @@ -337,7 +349,7 @@ class Instrumentinspection(Base): performedBy = Column(String(255)) status = Column(String(255)) remarks = Column(String(255)) - performedAt = Column(ForeignKey('station.stationId'), index=True) + performedAt = Column(ForeignKey('station.stationId')) station = relationship('Station') instrument = relationship('Instrument') @@ -362,14 +374,16 @@ class Physicalfeature(Base): __tablename__ = 'physicalfeature' __table_args__ = ( Index('featureIdentification', 'associatedWith', 'beginDate', 'classifiedInto', 'description', unique=True), + Index('physicalFeatureidentification_idx', 'classifiedInto'), + Index('stationfeature', 'associatedWith') ) - associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) image = Column(String(255)) description = Column(String(255), primary_key=True, nullable=False) - classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False, index=True) + classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False) station = relationship('Station') physicalfeatureclas = relationship('Physicalfeatureclas') @@ -379,13 +393,15 @@ class Stationelement(Base): __tablename__ = 'stationelement' __table_args__ = ( Index('stationElementIdentification', 'recordedFrom', 'describedBy', 'recordedWith', 'beginDate', unique=True), + Index('obsElementobservationInitial', 'describedBy'), + Index('stationobservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) - recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) + recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False) instrumentcode = Column(String(6)) - scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass'), index=True) + scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass')) height = Column(Float(6)) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) @@ -404,7 +420,7 @@ class Faultresolution(Base): resolvedDatetime = Column(String(50), primary_key=True, nullable=False) resolvedBy = Column(String(255)) - associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False, index=True) + associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False) remarks = Column(String(255)) instrumentfaultreport = relationship('Instrumentfaultreport') diff --git a/requirements.txt b/requirements.txt index 028cc477..714a9e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ psycopg2==2.9.1 sqlacodegen==2.3.0 SQLAlchemy==1.4.22 mysqlclient==2.0.3 +pytest==6.2.5 From b4705cd49b68743fbe53fd926dfe806f36cf28c0 Mon Sep 17 00:00:00 2001 From: Shaibu Shaibu Date: Wed, 8 Sep 2021 23:11:42 +0100 Subject: [PATCH 4/7] use mysql double column type for latitude and longitude columns --- opencdms/models/climsoft/v4_1_1_core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/opencdms/models/climsoft/v4_1_1_core.py b/opencdms/models/climsoft/v4_1_1_core.py index 9e84a75a..d9a098b3 100644 --- a/opencdms/models/climsoft/v4_1_1_core.py +++ b/opencdms/models/climsoft/v4_1_1_core.py @@ -1,6 +1,6 @@ # coding: utf-8 from sqlalchemy import BigInteger, CHAR, Column, DECIMAL, DateTime, Float, ForeignKey, Index, Integer, String, Text, text -from sqlalchemy.dialects.mysql import TINYINT +from sqlalchemy.dialects.mysql import TINYINT, DOUBLE from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -122,9 +122,9 @@ class Station(Base): stationName = Column(String(255)) wmoid = Column(String(20)) icaoid = Column(String(20)) - latitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) qualifier = Column(String(20)) - longitude = Column(Float(11, True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) elevation = Column(String(255)) geoLocationMethod = Column(String(255)) geoLocationAccuracy = Column(Float(11)) @@ -154,8 +154,8 @@ class Featuregeographicalposition(Base): belongsTo = Column(ForeignKey('synopfeature.abbreviation'), primary_key=True, nullable=False) observedOn = Column(String(50), primary_key=True, nullable=False) - latitude = Column(Float(11, True)) - longitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) synopfeature = relationship('Synopfeature') @@ -291,8 +291,8 @@ class Stationlocationhistory(Base): geoLocationAccuracy = Column(Float(11)) openingDatetime = Column(String(50), primary_key=True, nullable=False) closingDatetime = Column(String(50)) - latitude = Column(Float(11, True)) - longitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) elevation = Column(BigInteger) authority = Column(String(255)) adminRegion = Column(String(255)) From 1ad710ef35af9b515e9c28d08ad2481d2481e168 Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Fri, 10 Sep 2021 00:56:30 +0600 Subject: [PATCH 5/7] to see the diff in climsoft models --- opencdms/models/climsoft/v4_1_1_core.py | 108 +++++++----------------- 1 file changed, 32 insertions(+), 76 deletions(-) diff --git a/opencdms/models/climsoft/v4_1_1_core.py b/opencdms/models/climsoft/v4_1_1_core.py index d9a098b3..cd1479f2 100644 --- a/opencdms/models/climsoft/v4_1_1_core.py +++ b/opencdms/models/climsoft/v4_1_1_core.py @@ -1,6 +1,6 @@ # coding: utf-8 from sqlalchemy import BigInteger, CHAR, Column, DECIMAL, DateTime, Float, ForeignKey, Index, Integer, String, Text, text -from sqlalchemy.dialects.mysql import TINYINT, DOUBLE +from sqlalchemy.dialects.mysql import TINYINT from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -8,34 +8,6 @@ metadata = Base.metadata -TARGET_TABLES = [ - "acquisitiontype", - "data_forms", - "flags", - "obselement", - "paperarchivedefinition", - "qcstatusdefinition", - "qctype", - "regkeys", - "station", - "synopfeature", - "featuregeographicalposition", - "instrument", - "observationfinal", - "observationinitial", - "obsscheduleclass", - "paperarchive", - "physicalfeatureclass", - "stationlocationhistory", - "stationqualifier", - "instrumentfaultreport", - "instrumentinspection", - "observationschedule", - "physicalfeature", - "stationelement", - "faultresolution" -] - class Acquisitiontype(Base): __tablename__ = 'acquisitiontype' @@ -69,9 +41,8 @@ class Flag(Base): class Obselement(Base): __tablename__ = 'obselement' - __table_args__ = ( Index('elementCode', 'elementId'),) - elementId = Column(BigInteger, primary_key=True, server_default=text("'0'")) + elementId = Column(BigInteger, primary_key=True, index=True, server_default=text("'0'")) abbreviation = Column(String(255)) elementName = Column(String(255)) description = Column(String(255)) @@ -86,9 +57,8 @@ class Obselement(Base): class Paperarchivedefinition(Base): __tablename__ = 'paperarchivedefinition' - __table_args__ = (Index('paperarchivedef','formId'),) - formId = Column(String(50), primary_key=True) + formId = Column(String(50), primary_key=True, index=True) description = Column(String(255)) @@ -116,15 +86,14 @@ class Regkey(Base): class Station(Base): __tablename__ = 'station' - __table_args__ = (Index('StationStationId', 'stationId'),) - stationId = Column(String(255), primary_key=True) + stationId = Column(String(255), primary_key=True, index=True) stationName = Column(String(255)) wmoid = Column(String(20)) icaoid = Column(String(20)) - latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + latitude = Column(Float(11, True)) qualifier = Column(String(20)) - longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + longitude = Column(Float(11, True)) elevation = Column(String(255)) geoLocationMethod = Column(String(255)) geoLocationAccuracy = Column(Float(11)) @@ -154,18 +123,17 @@ class Featuregeographicalposition(Base): belongsTo = Column(ForeignKey('synopfeature.abbreviation'), primary_key=True, nullable=False) observedOn = Column(String(50), primary_key=True, nullable=False) - latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) - longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + latitude = Column(Float(11, True)) + longitude = Column(Float(11, True)) synopfeature = relationship('Synopfeature') class Instrument(Base): __tablename__ = 'instrument' - __table_args__ = (Index('code', 'instrumentId'),) instrumentName = Column(String(255)) - instrumentId = Column(String(255), primary_key=True) + instrumentId = Column(String(255), primary_key=True, index=True) serialNumber = Column(String(255)) abbreviation = Column(String(255)) model = Column(String(255)) @@ -175,7 +143,7 @@ class Instrument(Base): deinstallationDatetime = Column(String(50)) height = Column(String(255)) instrumentPicture = Column(CHAR(255)) - installedAt = Column(ForeignKey('station.stationId')) + installedAt = Column(ForeignKey('station.stationId'), index=True) station = relationship('Station') @@ -184,12 +152,10 @@ class Observationfinal(Base): __tablename__ = 'observationfinal' __table_args__ = ( Index('obsFinalIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', unique=True), - Index('obsElementObservationInitial', 'describedBy'), - Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False, server_default=text("'surface'")) obsValue = Column(DECIMAL(8, 2)) @@ -215,12 +181,10 @@ class Observationinitial(Base): __tablename__ = 'observationinitial' __table_args__ = ( Index('obsInitialIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', 'qcStatus', 'acquisitionType', unique=True), - Index('obsElementObservationInitial', 'describedBy'), - Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False) obsValue = Column(String(255)) @@ -244,11 +208,10 @@ class Observationinitial(Base): class Obsscheduleclas(Base): __tablename__ = 'obsscheduleclass' - __table_args__ = (Index('scheduleClassIdeification', 'scheduleClass'),) - scheduleClass = Column(String(255), primary_key=True, server_default=text("''")) + scheduleClass = Column(String(255), primary_key=True, index=True, server_default=text("''")) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId')) + refersTo = Column(ForeignKey('station.stationId'), index=True) station = relationship('Station') @@ -262,7 +225,7 @@ class Paperarchive(Base): belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) formDatetime = Column(DateTime, primary_key=True, nullable=False) image = Column(String(255)) - classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False) + classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False, index=True) station = relationship('Station') paperarchivedefinition = relationship('Paperarchivedefinition') @@ -270,11 +233,10 @@ class Paperarchive(Base): class Physicalfeatureclas(Base): __tablename__ = 'physicalfeatureclass' - __table_args__ = (Index('stationFeatureClass', 'featureClass'),) - featureClass = Column(String(255), primary_key=True) + featureClass = Column(String(255), primary_key=True, index=True) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId')) + refersTo = Column(ForeignKey('station.stationId'), index=True) station = relationship('Station') @@ -291,8 +253,8 @@ class Stationlocationhistory(Base): geoLocationAccuracy = Column(Float(11)) openingDatetime = Column(String(50), primary_key=True, nullable=False) closingDatetime = Column(String(50)) - latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) - longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + latitude = Column(Float(11, True)) + longitude = Column(Float(11, True)) elevation = Column(BigInteger) authority = Column(String(255)) adminRegion = Column(String(255)) @@ -305,7 +267,6 @@ class Stationqualifier(Base): __tablename__ = 'stationqualifier' __table_args__ = ( Index('stationid_qualifier_identification', 'qualifier', 'qualifierBeginDate', 'qualifierEndDate', 'belongsTo', unique=True), - Index('stationQualifierIdentification', 'belongsTo') ) qualifier = Column(String(255), primary_key=True, nullable=False) @@ -313,7 +274,7 @@ class Stationqualifier(Base): qualifierEndDate = Column(String(50), primary_key=True, nullable=False) stationTimeZone = Column(Integer, server_default=text("'0'")) stationNetworkType = Column(String(255)) - belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) station = relationship('Station') @@ -322,17 +283,16 @@ class Instrumentfaultreport(Base): __tablename__ = 'instrumentfaultreport' __table_args__ = ( Index('instrument_report', 'refersTo', 'reportDatetime', 'reportedFrom', unique=True), - Index('report_id', 'reportId') ) refersTo = Column(ForeignKey('instrument.instrumentId')) - reportId = Column(BigInteger, primary_key=True) + reportId = Column(BigInteger, primary_key=True, index=True) reportDatetime = Column(String(50)) faultDescription = Column(String(255)) reportedBy = Column(String(255)) receivedDatetime = Column(String(50)) receivedBy = Column(String(255)) - reportedFrom = Column(ForeignKey('station.stationId')) + reportedFrom = Column(ForeignKey('station.stationId'), index=True) instrument = relationship('Instrument') station = relationship('Station') @@ -349,7 +309,7 @@ class Instrumentinspection(Base): performedBy = Column(String(255)) status = Column(String(255)) remarks = Column(String(255)) - performedAt = Column(ForeignKey('station.stationId')) + performedAt = Column(ForeignKey('station.stationId'), index=True) station = relationship('Station') instrument = relationship('Instrument') @@ -374,16 +334,14 @@ class Physicalfeature(Base): __tablename__ = 'physicalfeature' __table_args__ = ( Index('featureIdentification', 'associatedWith', 'beginDate', 'classifiedInto', 'description', unique=True), - Index('physicalFeatureidentification_idx', 'classifiedInto'), - Index('stationfeature', 'associatedWith') ) - associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) image = Column(String(255)) description = Column(String(255), primary_key=True, nullable=False) - classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False) + classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False, index=True) station = relationship('Station') physicalfeatureclas = relationship('Physicalfeatureclas') @@ -393,15 +351,13 @@ class Stationelement(Base): __tablename__ = 'stationelement' __table_args__ = ( Index('stationElementIdentification', 'recordedFrom', 'describedBy', 'recordedWith', 'beginDate', unique=True), - Index('obsElementobservationInitial', 'describedBy'), - Index('stationobservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) - recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False, index=True) instrumentcode = Column(String(6)) - scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass')) + scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass'), index=True) height = Column(Float(6)) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) @@ -420,7 +376,7 @@ class Faultresolution(Base): resolvedDatetime = Column(String(50), primary_key=True, nullable=False) resolvedBy = Column(String(255)) - associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False) + associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False, index=True) remarks = Column(String(255)) instrumentfaultreport = relationship('Instrumentfaultreport') From b745532553e5cbe08e7caef0bb6f7ca3e765c455 Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Fri, 10 Sep 2021 01:50:59 +0600 Subject: [PATCH 6/7] Revert "to see the diff in climsoft models" This reverts commit 1ad710ef35af9b515e9c28d08ad2481d2481e168. --- opencdms/models/climsoft/v4_1_1_core.py | 108 +++++++++++++++++------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/opencdms/models/climsoft/v4_1_1_core.py b/opencdms/models/climsoft/v4_1_1_core.py index cd1479f2..d9a098b3 100644 --- a/opencdms/models/climsoft/v4_1_1_core.py +++ b/opencdms/models/climsoft/v4_1_1_core.py @@ -1,6 +1,6 @@ # coding: utf-8 from sqlalchemy import BigInteger, CHAR, Column, DECIMAL, DateTime, Float, ForeignKey, Index, Integer, String, Text, text -from sqlalchemy.dialects.mysql import TINYINT +from sqlalchemy.dialects.mysql import TINYINT, DOUBLE from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -8,6 +8,34 @@ metadata = Base.metadata +TARGET_TABLES = [ + "acquisitiontype", + "data_forms", + "flags", + "obselement", + "paperarchivedefinition", + "qcstatusdefinition", + "qctype", + "regkeys", + "station", + "synopfeature", + "featuregeographicalposition", + "instrument", + "observationfinal", + "observationinitial", + "obsscheduleclass", + "paperarchive", + "physicalfeatureclass", + "stationlocationhistory", + "stationqualifier", + "instrumentfaultreport", + "instrumentinspection", + "observationschedule", + "physicalfeature", + "stationelement", + "faultresolution" +] + class Acquisitiontype(Base): __tablename__ = 'acquisitiontype' @@ -41,8 +69,9 @@ class Flag(Base): class Obselement(Base): __tablename__ = 'obselement' + __table_args__ = ( Index('elementCode', 'elementId'),) - elementId = Column(BigInteger, primary_key=True, index=True, server_default=text("'0'")) + elementId = Column(BigInteger, primary_key=True, server_default=text("'0'")) abbreviation = Column(String(255)) elementName = Column(String(255)) description = Column(String(255)) @@ -57,8 +86,9 @@ class Obselement(Base): class Paperarchivedefinition(Base): __tablename__ = 'paperarchivedefinition' + __table_args__ = (Index('paperarchivedef','formId'),) - formId = Column(String(50), primary_key=True, index=True) + formId = Column(String(50), primary_key=True) description = Column(String(255)) @@ -86,14 +116,15 @@ class Regkey(Base): class Station(Base): __tablename__ = 'station' + __table_args__ = (Index('StationStationId', 'stationId'),) - stationId = Column(String(255), primary_key=True, index=True) + stationId = Column(String(255), primary_key=True) stationName = Column(String(255)) wmoid = Column(String(20)) icaoid = Column(String(20)) - latitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) qualifier = Column(String(20)) - longitude = Column(Float(11, True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) elevation = Column(String(255)) geoLocationMethod = Column(String(255)) geoLocationAccuracy = Column(Float(11)) @@ -123,17 +154,18 @@ class Featuregeographicalposition(Base): belongsTo = Column(ForeignKey('synopfeature.abbreviation'), primary_key=True, nullable=False) observedOn = Column(String(50), primary_key=True, nullable=False) - latitude = Column(Float(11, True)) - longitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) synopfeature = relationship('Synopfeature') class Instrument(Base): __tablename__ = 'instrument' + __table_args__ = (Index('code', 'instrumentId'),) instrumentName = Column(String(255)) - instrumentId = Column(String(255), primary_key=True, index=True) + instrumentId = Column(String(255), primary_key=True) serialNumber = Column(String(255)) abbreviation = Column(String(255)) model = Column(String(255)) @@ -143,7 +175,7 @@ class Instrument(Base): deinstallationDatetime = Column(String(50)) height = Column(String(255)) instrumentPicture = Column(CHAR(255)) - installedAt = Column(ForeignKey('station.stationId'), index=True) + installedAt = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -152,10 +184,12 @@ class Observationfinal(Base): __tablename__ = 'observationfinal' __table_args__ = ( Index('obsFinalIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', unique=True), + Index('obsElementObservationInitial', 'describedBy'), + Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False, server_default=text("'surface'")) obsValue = Column(DECIMAL(8, 2)) @@ -181,10 +215,12 @@ class Observationinitial(Base): __tablename__ = 'observationinitial' __table_args__ = ( Index('obsInitialIdentification', 'recordedFrom', 'describedBy', 'obsDatetime', 'qcStatus', 'acquisitionType', unique=True), + Index('obsElementObservationInitial', 'describedBy'), + Index('stationObservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) obsDatetime = Column(DateTime, primary_key=True, nullable=False) obsLevel = Column(String(255), primary_key=True, nullable=False) obsValue = Column(String(255)) @@ -208,10 +244,11 @@ class Observationinitial(Base): class Obsscheduleclas(Base): __tablename__ = 'obsscheduleclass' + __table_args__ = (Index('scheduleClassIdeification', 'scheduleClass'),) - scheduleClass = Column(String(255), primary_key=True, index=True, server_default=text("''")) + scheduleClass = Column(String(255), primary_key=True, server_default=text("''")) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId'), index=True) + refersTo = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -225,7 +262,7 @@ class Paperarchive(Base): belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) formDatetime = Column(DateTime, primary_key=True, nullable=False) image = Column(String(255)) - classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False, index=True) + classifiedInto = Column(ForeignKey('paperarchivedefinition.formId'), primary_key=True, nullable=False) station = relationship('Station') paperarchivedefinition = relationship('Paperarchivedefinition') @@ -233,10 +270,11 @@ class Paperarchive(Base): class Physicalfeatureclas(Base): __tablename__ = 'physicalfeatureclass' + __table_args__ = (Index('stationFeatureClass', 'featureClass'),) - featureClass = Column(String(255), primary_key=True, index=True) + featureClass = Column(String(255), primary_key=True) description = Column(String(255)) - refersTo = Column(ForeignKey('station.stationId'), index=True) + refersTo = Column(ForeignKey('station.stationId')) station = relationship('Station') @@ -253,8 +291,8 @@ class Stationlocationhistory(Base): geoLocationAccuracy = Column(Float(11)) openingDatetime = Column(String(50), primary_key=True, nullable=False) closingDatetime = Column(String(50)) - latitude = Column(Float(11, True)) - longitude = Column(Float(11, True)) + latitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) + longitude = Column(DOUBLE(precision=11, scale=6, asdecimal=True)) elevation = Column(BigInteger) authority = Column(String(255)) adminRegion = Column(String(255)) @@ -267,6 +305,7 @@ class Stationqualifier(Base): __tablename__ = 'stationqualifier' __table_args__ = ( Index('stationid_qualifier_identification', 'qualifier', 'qualifierBeginDate', 'qualifierEndDate', 'belongsTo', unique=True), + Index('stationQualifierIdentification', 'belongsTo') ) qualifier = Column(String(255), primary_key=True, nullable=False) @@ -274,7 +313,7 @@ class Stationqualifier(Base): qualifierEndDate = Column(String(50), primary_key=True, nullable=False) stationTimeZone = Column(Integer, server_default=text("'0'")) stationNetworkType = Column(String(255)) - belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + belongsTo = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) station = relationship('Station') @@ -283,16 +322,17 @@ class Instrumentfaultreport(Base): __tablename__ = 'instrumentfaultreport' __table_args__ = ( Index('instrument_report', 'refersTo', 'reportDatetime', 'reportedFrom', unique=True), + Index('report_id', 'reportId') ) refersTo = Column(ForeignKey('instrument.instrumentId')) - reportId = Column(BigInteger, primary_key=True, index=True) + reportId = Column(BigInteger, primary_key=True) reportDatetime = Column(String(50)) faultDescription = Column(String(255)) reportedBy = Column(String(255)) receivedDatetime = Column(String(50)) receivedBy = Column(String(255)) - reportedFrom = Column(ForeignKey('station.stationId'), index=True) + reportedFrom = Column(ForeignKey('station.stationId')) instrument = relationship('Instrument') station = relationship('Station') @@ -309,7 +349,7 @@ class Instrumentinspection(Base): performedBy = Column(String(255)) status = Column(String(255)) remarks = Column(String(255)) - performedAt = Column(ForeignKey('station.stationId'), index=True) + performedAt = Column(ForeignKey('station.stationId')) station = relationship('Station') instrument = relationship('Instrument') @@ -334,14 +374,16 @@ class Physicalfeature(Base): __tablename__ = 'physicalfeature' __table_args__ = ( Index('featureIdentification', 'associatedWith', 'beginDate', 'classifiedInto', 'description', unique=True), + Index('physicalFeatureidentification_idx', 'classifiedInto'), + Index('stationfeature', 'associatedWith') ) - associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) + associatedWith = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) image = Column(String(255)) description = Column(String(255), primary_key=True, nullable=False) - classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False, index=True) + classifiedInto = Column(ForeignKey('physicalfeatureclass.featureClass'), primary_key=True, nullable=False) station = relationship('Station') physicalfeatureclas = relationship('Physicalfeatureclas') @@ -351,13 +393,15 @@ class Stationelement(Base): __tablename__ = 'stationelement' __table_args__ = ( Index('stationElementIdentification', 'recordedFrom', 'describedBy', 'recordedWith', 'beginDate', unique=True), + Index('obsElementobservationInitial', 'describedBy'), + Index('stationobservationInitial', 'recordedFrom') ) - recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False, index=True) - describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False, index=True) - recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False, index=True) + recordedFrom = Column(ForeignKey('station.stationId'), primary_key=True, nullable=False) + describedBy = Column(ForeignKey('obselement.elementId'), primary_key=True, nullable=False) + recordedWith = Column(ForeignKey('instrument.instrumentId'), primary_key=True, nullable=False) instrumentcode = Column(String(6)) - scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass'), index=True) + scheduledFor = Column(ForeignKey('obsscheduleclass.scheduleClass')) height = Column(Float(6)) beginDate = Column(String(50), primary_key=True, nullable=False) endDate = Column(String(50)) @@ -376,7 +420,7 @@ class Faultresolution(Base): resolvedDatetime = Column(String(50), primary_key=True, nullable=False) resolvedBy = Column(String(255)) - associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False, index=True) + associatedWith = Column(ForeignKey('instrumentfaultreport.reportId'), primary_key=True, nullable=False) remarks = Column(String(255)) instrumentfaultreport = relationship('Instrumentfaultreport') From de6a678b047c0ca1b9351e598e5d71222220273c Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Sat, 11 Sep 2021 12:48:38 +0600 Subject: [PATCH 7/7] remove some code --- .gitignore | 1 - opencdms/models/__init__.py | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/.gitignore b/.gitignore index 7a4a5233..4c915d14 100644 --- a/.gitignore +++ b/.gitignore @@ -104,4 +104,3 @@ ENV/ # IDE settings .vscode/ .idea/ -shaibu-activity.log diff --git a/opencdms/models/__init__.py b/opencdms/models/__init__.py index 485db632..c9349483 100644 --- a/opencdms/models/__init__.py +++ b/opencdms/models/__init__.py @@ -97,14 +97,3 @@ def get_schema_diff( diff = list(filter(filter_in_included_tables(include_tables), diff)) return list(diff) - -# TODO remove and use automated tests instead. -if __name__ == "__main__": - from opencdms.models.climsoft.v4_1_1_core import metadata, TARGET_TABLES - diff = get_schema_diff( - metadata, - "mysql://root:#Classicnerdswag1@localhost/mariadb_climsoft_db_v4", - include_tables=TARGET_TABLES - ) - with open('output.txt', 'wt') as out: - pprint(diff, stream=out, indent=4)