more error handling for frame files

NEONScience · Dec 4, 2024 · a75fd4a · a75fd4a
1 parent 4619bad
commit a75fd4a
Show file tree

Hide file tree

Showing 10 changed files with 13 additions and 1 deletion.
diff --git a/dist/neonutilities-1.0.1-py3-none-any.whl b/dist/neonutilities-1.0.1-py3-none-any.whl
diff --git a/dist/neonutilities-1.0.1.tar.gz b/dist/neonutilities-1.0.1.tar.gz
diff --git a/src/neonutilities/unzip_and_stack.py b/src/neonutilities/unzip_and_stack.py
@@ -546,7 +546,19 @@ def stack_frame_files(framefiles, dpid,
         fdat = dataset.dataset(source=framefiles, format="csv",
                                schema=fschema)
 
-    fdattab = fdat.to_table()
+    try:
+        fdattab = fdat.to_table()
+    except Exception:
+        stringschema = unknown_string_schema(fdat.head(num_rows=0).column_names)
+        if cloud_mode:
+            fdat = dataset.dataset(source=framebuckets, filesystem=gcs, 
+                                   format="csv", schema=stringschema)
+        else:
+            fdat = dataset.dataset(source=framefiles,
+                                   format="csv", schema=stringschema)
+        fdattab = fdat.to_table()
+        logging.info("Large file schema did not match expectations; all variable types set to string.")
+
     fpdat = fdattab.to_pandas()
 
     nm = "per_sample"

diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc
diff --git a/tests/__pycache__/test_aop_download.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_aop_download.cpython-311-pytest-8.1.1.pyc
diff --git a/tests/__pycache__/test_get_citation.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_get_citation.cpython-311-pytest-8.1.1.pyc
diff --git a/tests/__pycache__/test_get_issue_log.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_get_issue_log.cpython-311-pytest-8.1.1.pyc
diff --git a/tests/__pycache__/test_load_by_product.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_load_by_product.cpython-311-pytest-8.1.1.pyc
diff --git a/tests/__pycache__/test_stack_by_table.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_stack_by_table.cpython-311-pytest-8.1.1.pyc
diff --git a/tests/__pycache__/test_zips_by_product.cpython-311-pytest-8.1.1.pyc b/tests/__pycache__/test_zips_by_product.cpython-311-pytest-8.1.1.pyc