Continue work on survey_tally.py - testing .csv generation

.vscode/launch.json

@@ -86,7 +86,8 @@
       "program": "${workspaceFolder}/smdb/scripts/survey_tally.py",
       "console": "integratedTerminal",
       //"args": ["-v", "2", "--parent_dir", "2023", "--read_xlsx"],
-      "args": ["-v", "2", "--parent_dir", "2023", "--write_csv"],
+      //"args": ["-v", "2", "--parent_dir", "2023", "--write_csv"],
+      "args": ["-v", "1", "--write_csv"],
     }
   ]
 }

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "python.pythonPath": "/bin/python",
   "terminal.integrated.defaultProfile.linux": "zsh",
-  "python.linting.mypyCategorySeverity.error": "Hint"
-}
+  "python.linting.mypyCategorySeverity.error": "Hint",
+  "github-actions.workflows.pinned.workflows": []
+}

smdb/compose/production/django/Dockerfile

@@ -4,10 +4,7 @@ ARG DOCKER_USER_ID
 ARG APP_HOME=/app
 WORKDIR ${APP_HOME}
 
-COPY ./package.json ${APP_HOME}
-RUN npm install && npm cache clean --force
 COPY . ${APP_HOME}
-RUN npm run build
 
 # Base the build off of recent stable GDAL image
 FROM osgeo/gdal:ubuntu-small-3.6.2 as python

smdb/scripts/survey_tally.py

@@ -132,26 +132,41 @@ def process_xlsx(self) -> None:
         self.update_db_from_df(df)
 
     def get_parent_dirs(self) -> List[str]:
+        """Return a list of parent directories to process. Check if they are in the database.
+        If a parent_dir is specified, only return that one. Omit the '/mbari/SeafloorMapping/' prefix.
+        """
+        parent_dirs_in_db = {
+            m.split("/")[0]
+            for m in Mission.objects.values_list("name", flat=True).distinct()
+        }
+        parent_dirs = []
         if self.args.parent_dir:
             if os.path.isdir(os.path.join(MBARI_DIR, self.args.parent_dir)):
-                return [self.args.parent_dir]
+                if self.args.parent_dir in parent_dirs_in_db:
+                    parent_dirs.append(self.args.parent_dir)
+                else:
+                    self.logger.warning(
+                        f"Directory {self.args.parent_dir} not found in database {os.environ['DATABASE_URL']}"
+                    )
             else:
-                print(f"Directory {self.args.parent_dir} not found in {MBARI_DIR}")
-                sys.exit(1)
+                self.logger.warning(
+                    f"Directory {self.args.parent_dir} not found in {MBARI_DIR}"
+                )
         else:
-            return [
-                f
-                for f in os.listdir(MBARI_DIR)
-                if os.path.isdir(os.path.join(MBARI_DIR, f))
-            ]
-        return os.path.join(MBARI_DIR, self.args.parent_dir)
+            for f in os.listdir(MBARI_DIR):
+                if os.path.isdir(os.path.join(MBARI_DIR, f)):
+                    if f in parent_dirs_in_db:
+                        parent_dirs.append(f)
+                    else:
+                        self.logger.debug(f"Directory {f} not found in database")
+        return parent_dirs
 
     def read_from_db_into_rows(self, parent_dir: str) -> pd.DataFrame:
         # cols must match field names in the Mission table - to be cols in the .csv file
         cols = [
             "name",  # Saved without the parent_dir suffix
             "route_file",
-            "location",  # Location is a foreign key to Location table
+            "location",
             "vehicle",
             "quality_comment",
             "auv",
@@ -171,20 +186,23 @@ def read_from_db_into_rows(self, parent_dir: str) -> pd.DataFrame:
                 if col == "name":
                     item = getattr(mission, col).replace(f"{parent_dir}/", "")
                 else:
-                    item = getattr(mission, col, "")
+                    if hasattr(mission, col):
+                        item = getattr(mission, col, "") or ""
+                    else:
+                        self.logger.warning(f"Mission model missing field: {col}")
+                        item = ""
                 row.append(str(item))
             rows.append(row)
         return cols, rows
 
     def process_csv(self):
         for parent_dir in self.get_parent_dirs():
             cols, rows = self.read_from_db_into_rows(parent_dir)
-            csv_file = os.path.join(
-                MBARI_DIR,
-                parent_dir,
-                "SMDB",
-                f"SMDB_{parent_dir}_survey_tally.csv",
-            )
+            # csv_dir = os.path.join(MBARI_DIR, parent_dir, "SMDB")
+            csv_dir = os.path.join("/tmp/SeafloorMapping", parent_dir, "SMDB")
+            if not os.path.exists(csv_dir):
+                os.makedirs(csv_dir)
+            csv_file = os.path.join(csv_dir, f"SMDB_{parent_dir}_survey_tally.csv")
             self.logger.info(f"Writing {csv_file}")
             with open(csv_file, "w") as f:
                 f.write(",".join(cols) + "\n")

smdb/smdb/tables.py

@@ -19,6 +19,7 @@ def render(self, value):
 class MissionTable(Table):
     name = Column(linkify=True)
     expedition = Column(linkify=True)
+    compilation_set = ManyToManyColumn(linkify_item=True, verbose_name="Compilations")
 
     class Meta:
         model = Mission
@@ -27,6 +28,13 @@ class Meta:
             "track_length",
             "start_depth",
         )
+        sequence = (
+            "name",
+            "track_length",
+            "start_depth",
+            "expedition",
+            "compilation_set",
+        )
 
 
 class ExpeditionTable(Table):

smdb/smdb/views.py

@@ -242,6 +242,7 @@ def get_context_data(self, **kwargs):
 
         table = MissionTable(
             Mission.objects.filter(expedition=expedition),
+            exclude=["expedition"],
         )
         RequestConfig(self.request).configure(table)
         context["table"] = table