diff --git a/.vscode/launch.json b/.vscode/launch.json index 231c0c47..c8595a76 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -86,7 +86,8 @@ "program": "${workspaceFolder}/smdb/scripts/survey_tally.py", "console": "integratedTerminal", //"args": ["-v", "2", "--parent_dir", "2023", "--read_xlsx"], - "args": ["-v", "2", "--parent_dir", "2023", "--write_csv"], + //"args": ["-v", "2", "--parent_dir", "2023", "--write_csv"], + "args": ["-v", "1", "--write_csv"], } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 54c026f5..0bad2523 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "python.pythonPath": "/bin/python", "terminal.integrated.defaultProfile.linux": "zsh", - "python.linting.mypyCategorySeverity.error": "Hint" -} + "python.linting.mypyCategorySeverity.error": "Hint", + "github-actions.workflows.pinned.workflows": [] +} \ No newline at end of file diff --git a/smdb/compose/production/django/Dockerfile b/smdb/compose/production/django/Dockerfile index e41f1ce9..a77f16fb 100644 --- a/smdb/compose/production/django/Dockerfile +++ b/smdb/compose/production/django/Dockerfile @@ -4,10 +4,7 @@ ARG DOCKER_USER_ID ARG APP_HOME=/app WORKDIR ${APP_HOME} -COPY ./package.json ${APP_HOME} -RUN npm install && npm cache clean --force COPY . ${APP_HOME} -RUN npm run build # Base the build off of recent stable GDAL image FROM osgeo/gdal:ubuntu-small-3.6.2 as python diff --git a/smdb/scripts/survey_tally.py b/smdb/scripts/survey_tally.py index d030b0c7..b6aa0925 100755 --- a/smdb/scripts/survey_tally.py +++ b/smdb/scripts/survey_tally.py @@ -132,26 +132,41 @@ def process_xlsx(self) -> None: self.update_db_from_df(df) def get_parent_dirs(self) -> List[str]: + """Return a list of parent directories to process. Check if they are in the database. + If a parent_dir is specified, only return that one. Omit the '/mbari/SeafloorMapping/' prefix. + """ + parent_dirs_in_db = { + m.split("/")[0] + for m in Mission.objects.values_list("name", flat=True).distinct() + } + parent_dirs = [] if self.args.parent_dir: if os.path.isdir(os.path.join(MBARI_DIR, self.args.parent_dir)): - return [self.args.parent_dir] + if self.args.parent_dir in parent_dirs_in_db: + parent_dirs.append(self.args.parent_dir) + else: + self.logger.warning( + f"Directory {self.args.parent_dir} not found in database {os.environ['DATABASE_URL']}" + ) else: - print(f"Directory {self.args.parent_dir} not found in {MBARI_DIR}") - sys.exit(1) + self.logger.warning( + f"Directory {self.args.parent_dir} not found in {MBARI_DIR}" + ) else: - return [ - f - for f in os.listdir(MBARI_DIR) - if os.path.isdir(os.path.join(MBARI_DIR, f)) - ] - return os.path.join(MBARI_DIR, self.args.parent_dir) + for f in os.listdir(MBARI_DIR): + if os.path.isdir(os.path.join(MBARI_DIR, f)): + if f in parent_dirs_in_db: + parent_dirs.append(f) + else: + self.logger.debug(f"Directory {f} not found in database") + return parent_dirs def read_from_db_into_rows(self, parent_dir: str) -> pd.DataFrame: # cols must match field names in the Mission table - to be cols in the .csv file cols = [ "name", # Saved without the parent_dir suffix "route_file", - "location", # Location is a foreign key to Location table + "location", "vehicle", "quality_comment", "auv", @@ -171,7 +186,11 @@ def read_from_db_into_rows(self, parent_dir: str) -> pd.DataFrame: if col == "name": item = getattr(mission, col).replace(f"{parent_dir}/", "") else: - item = getattr(mission, col, "") + if hasattr(mission, col): + item = getattr(mission, col, "") or "" + else: + self.logger.warning(f"Mission model missing field: {col}") + item = "" row.append(str(item)) rows.append(row) return cols, rows @@ -179,12 +198,11 @@ def read_from_db_into_rows(self, parent_dir: str) -> pd.DataFrame: def process_csv(self): for parent_dir in self.get_parent_dirs(): cols, rows = self.read_from_db_into_rows(parent_dir) - csv_file = os.path.join( - MBARI_DIR, - parent_dir, - "SMDB", - f"SMDB_{parent_dir}_survey_tally.csv", - ) + # csv_dir = os.path.join(MBARI_DIR, parent_dir, "SMDB") + csv_dir = os.path.join("/tmp/SeafloorMapping", parent_dir, "SMDB") + if not os.path.exists(csv_dir): + os.makedirs(csv_dir) + csv_file = os.path.join(csv_dir, f"SMDB_{parent_dir}_survey_tally.csv") self.logger.info(f"Writing {csv_file}") with open(csv_file, "w") as f: f.write(",".join(cols) + "\n") diff --git a/smdb/smdb/tables.py b/smdb/smdb/tables.py index 6b2144d3..6da143e3 100644 --- a/smdb/smdb/tables.py +++ b/smdb/smdb/tables.py @@ -19,6 +19,7 @@ def render(self, value): class MissionTable(Table): name = Column(linkify=True) expedition = Column(linkify=True) + compilation_set = ManyToManyColumn(linkify_item=True, verbose_name="Compilations") class Meta: model = Mission @@ -27,6 +28,13 @@ class Meta: "track_length", "start_depth", ) + sequence = ( + "name", + "track_length", + "start_depth", + "expedition", + "compilation_set", + ) class ExpeditionTable(Table): diff --git a/smdb/smdb/views.py b/smdb/smdb/views.py index aa890fef..1a1fce80 100644 --- a/smdb/smdb/views.py +++ b/smdb/smdb/views.py @@ -242,6 +242,7 @@ def get_context_data(self, **kwargs): table = MissionTable( Mission.objects.filter(expedition=expedition), + exclude=["expedition"], ) RequestConfig(self.request).configure(table) context["table"] = table