bug fixes; add dataset_test for example datasets (#186)

* a few bug fixes, identified in part by @hrlai - further mismatches to function names, functions that are now in `austraits` were still listed in `_pkgdown.yml` * another issue uncovered by @hrlai indicated we really needed to run dataset_test on the traits.build example datasets: * added dataset_test for example datasets - includes allowing "output" folders and Readme files in data folders closes issue #183 closes issue #185 closes issue #143
traitecoevo · Dec 17, 2024 · 0f0517d · 0f0517d
1 parent ba2ada5
commit 0f0517d
Show file tree

Hide file tree

Showing 15 changed files with 280 additions and 239 deletions.
diff --git a/R/process.R b/R/process.R
@@ -708,6 +708,9 @@ process_format_contexts <- function(my_list, dataset_id, traits) {
       tibble::tibble(dataset_id = character(), var_in = character())
   }
 
+  contexts <- contexts %>% dplyr::select(dplyr::any_of(
+    c("dataset_id", "context_property", "category", "value", "description", "find", "var_in")))
+
   contexts
 }
 

diff --git a/R/testdata.R b/R/testdata.R
@@ -75,7 +75,7 @@ dataset_test_worker <-
         }
 
         ## Check for other files
-        vals <- c("data.csv", "metadata.yml", "raw")
+        vals <- c("data.csv", "metadata.yml", "raw", "output", "README.md")
         test_expect_is_in(
           dir(s), vals,
           info = paste0(red(file.path(path_data, dataset_id)), "\tdisallowed files"),
@@ -873,18 +873,20 @@ dataset_test_worker <-
           # I'm not testing whether the functions work as intended, just that they throw no error
 
           test_expect_no_warning(
-            dataset_wider <- db_traits_pivot_wider(dataset$traits),
-            info = paste0(red(dataset_id), "\t`db_traits_pivot_wider` threw a warning; duplicate rows detected")
+            dataset_wider <- check_pivot_wider(dataset),
+            info = paste0(red(dataset_id), "\t`check_pivot_wider` threw a warning; duplicate rows detected")
           )
 
-          if (exists("dataset_wider")) {
-            test_expect_no_warning(
-              test_expect_no_error(
-                dataset_longer <- db_traits_pivot_longer(dataset_wider),
-                info = paste0(red(dataset_id), "\t`db_traits_pivot_longer`")),
-              info = paste0(red(dataset_id), "\t`db_traits_pivot_longer` threw a warning")
-            )
-          }
+          # Commenting out test Dec 2024, because `check_pivot_longer` has been deprecated
+
+          #if (exists("dataset_wider")) {
+          #  test_expect_no_warning(
+          #    test_expect_no_error(
+          #      dataset_longer <- check_pivot_longer(dataset_wider),
+          #      info = paste0(red(dataset_id), "\t`check_pivot_longer`")),
+          #    info = paste0(red(dataset_id), "\t`check_pivot_longer` threw a warning")
+          #  )
+          #}
         }
       })
     }

diff --git a/_pkgdown.yml b/_pkgdown.yml
@@ -82,11 +82,8 @@ reference:
   - title: Working with outputs
     desc: Functions to work with compiled traits.build objects.
     contents:
-    - '`plot_trait_distribution_beeswarm`'
     - '`check_pivot_duplicates`'
     - '`check_pivot_wider`'
-    - '`db_traits_pivot_longer`'
-    - '`db_traits_pivot_wider`'
 
   - title: Building a compilation (private)
     desc: Private functions that are called in the background when building individual datasets and entire compilation. These are only of interest to developers.
@@ -127,12 +124,9 @@ reference:
     - '`util_check_all_values_in`'
     - '`util_check_disallowed_chars`'
     - '`util_df_convert_character`'
-    - '`util_df_to_list`'
     - '`util_extract_list_element`'
     - '`util_kable_styling_html`'
     - '`util_list_to_bib`'
-    - '`util_list_to_df1`'
-    - '`util_list_to_df2`'
     - '`util_replace_null`'
     - '`util_separate_and_sort`'
     - '`util_standardise_doi`'

diff --git a/inst/support/traits.build_schema.yml b/inst/support/traits.build_schema.yml
@@ -118,7 +118,7 @@ austraits:
         source_secondary_key: Citation key for the secondary source in `sources`.  The key is typically formatted as `Surname_year`.
         source_secondary_citation: Citations for the secondary source. This detail is generated from the secondary source in the metadata.
         source_original_dataset_key: Citation key for the original dataset_id in sources; for compilations.  The key is typically formatted as `Surname_year`.
-        source_original_dataset_citation: Citations for the original dataset_id in sources; for compilationse. This detail is generated from the original source in the metadata.
+        source_original_dataset_citation: Citations for the original dataset_id in sources; for compilations. This detail is generated from the original source in the metadata.
         data_collectors: &data_collectors The person (people) leading data collection for this study.
         assistants: &assistants Names of people who played a more minor role in data collection for the study.
         dataset_curators: &dataset_curators Names of database team member(s) who contacted the data collectors and added the study to the database repository.

diff --git a/tests/testthat/examples/Test_2023_1/metadata.yml b/tests/testthat/examples/Test_2023_1/metadata.yml
@@ -65,6 +65,8 @@ locations:
   Atherton:
     description: Tropical rain forest vegetation.
     elevation (m): 800
+    latitude (deg): -17.1166667
+    longitude (deg): 145.65
     rainfall (mm): 2000
   Cape Tribulation:
     description: Complex mesophyll vine forest in tropical rain forest.

diff --git a/tests/testthat/examples/Test_2023_1/output/locations.csv b/tests/testthat/examples/Test_2023_1/output/locations.csv
@@ -1,5 +1,7 @@
 dataset_id,location_id,location_name,location_property,value
 Test_2023_1,01,Atherton,description,Tropical rain forest vegetation.
+Test_2023_1,01,Atherton,latitude (deg),-17.1166667
+Test_2023_1,01,Atherton,longitude (deg),145.65
 Test_2023_1,01,Atherton,elevation (m),800
 Test_2023_1,01,Atherton,rainfall (mm),2000
 Test_2023_1,02,Cape Tribulation,description,Complex mesophyll vine forest in tropical rain forest.

diff --git a/tests/testthat/examples/Test_2023_2/metadata.yml b/tests/testthat/examples/Test_2023_2/metadata.yml
@@ -54,10 +54,14 @@ locations:
   Atherton:
     description: Tropical rain forest vegetation.
     elevation (m): 800
+    latitude (deg): -17.1166667
+    longitude (deg): 145.65
     rainfall (mm): 2000
   Cape Tribulation:
     description: Complex mesophyll vine forest in tropical rain forest.
     elevation (m): 25
+    latitude (deg): -16.1
+    longitude (deg): 145.45
     rainfall (mm): 3500
     life_stage: juvenile
     basis_of_record: field_experiment

diff --git a/tests/testthat/examples/Test_2023_2/output/locations.csv b/tests/testthat/examples/Test_2023_2/output/locations.csv
@@ -1,7 +1,11 @@
 dataset_id,location_id,location_name,location_property,value
 Test_2023_2,01,Atherton,description,Tropical rain forest vegetation.
+Test_2023_2,01,Atherton,latitude (deg),-17.1166667
+Test_2023_2,01,Atherton,longitude (deg),145.65
 Test_2023_2,01,Atherton,elevation (m),800
 Test_2023_2,01,Atherton,rainfall (mm),2000
 Test_2023_2,02,Cape Tribulation,description,Complex mesophyll vine forest in tropical rain forest.
+Test_2023_2,02,Cape Tribulation,latitude (deg),-16.1
+Test_2023_2,02,Cape Tribulation,longitude (deg),145.45
 Test_2023_2,02,Cape Tribulation,elevation (m),25
 Test_2023_2,02,Cape Tribulation,rainfall (mm),3500
diff --git a/tests/testthat/examples/Test_2023_3/metadata.yml b/tests/testthat/examples/Test_2023_3/metadata.yml
@@ -122,6 +122,8 @@ dataset:
   notes: .na
 locations:
   australia:
+    latitude (deg): -26.158
+    longitude (deg): 143.685
     description: First made-up location for testing
   australia_2:
     latitude (deg): -27.158

diff --git a/tests/testthat/examples/Test_2023_3/output/locations.csv b/tests/testthat/examples/Test_2023_3/output/locations.csv
@@ -1,5 +1,7 @@
 dataset_id,location_id,location_name,location_property,value
 Test_2023_3,01,australia,description,First made-up location for testing
+Test_2023_3,01,australia,latitude (deg),-26.158
+Test_2023_3,01,australia,longitude (deg),143.685
 Test_2023_3,02,australia_2,description,Second made-up location for testing
 Test_2023_3,02,australia_2,latitude (deg),-27.158
 Test_2023_3,02,australia_2,longitude (deg),142.685
diff --git a/tests/testthat/examples/Test_2023_4/README.md b/tests/testthat/examples/Test_2023_4/README.md
@@ -28,5 +28,4 @@ Test_2023_4 is a copy of NHNSW_2023 with the following modifications:
 - Added `leaf_photosynthesis` and `leaf_stomatal_conductance` to test `repeat_measurements_id`, with another `leaf_stomatal_conductance_2` variable that doesn't specify `repeat_measurements_id`
 - Duplicated `fruit_length` for Imperata cylindrica and Alectryon subdentatus to test `collection_date` making separate `observation_id`'s and added `collection_date` column to dataset-level metadata
 
-
 See output/ for expected output files.
diff --git a/tests/testthat/examples/Test_2023_4/metadata.yml b/tests/testthat/examples/Test_2023_4/metadata.yml
@@ -85,7 +85,10 @@ dataset:
         entity_measured = ifelse(category %in% c("capsule", "silicula", "nut", "utricles", "pyrenes", "berry", "achenes"), "fruit", entity_measured),
         units = ifelse(trait_name == "fruit_width", NA_character_, units)
       ) %>%
-      mutate(entity_measured = ifelse(is.na(entity_measured), "unknown", entity_measured))
+      mutate(entity_measured = ifelse(is.na(entity_measured), "unknown", entity_measured)) %>%
+      group_by(taxon_name, trait_name, entity_measured, value_type) %>%
+        mutate(observation_number = dplyr::row_number()) %>%
+      ungroup()
   '
   collection_date: collection_date
   taxon_name: taxon_name
@@ -150,6 +153,9 @@ contexts:
   values:
   - value: unknown
   - value: fruit
+- context_property: replicate observations
+  category: temporal_context
+  var_in: observation_number
 traits:
 - var_in: fruit_colour
   trait_name: fruit_colour

diff --git a/tests/testthat/examples/Test_2023_4/output/contexts.csv b/tests/testthat/examples/Test_2023_4/output/contexts.csv
@@ -5,3 +5,8 @@ Test_2023_4,trait scoring method,method_context,inferred_from_genus,Trait value
 Test_2023_4,trait scoring method,method_context,inferred_from_family,Trait value inferred from family-level taxon description.,method_context_id,01
 Test_2023_4,entity measured,entity_context,unknown,,entity_context_id,02
 Test_2023_4,entity measured,entity_context,fruit,,entity_context_id,01
+Test_2023_4,replicate observations,temporal_context,1,NA,temporal_context_id,01
+Test_2023_4,replicate observations,temporal_context,2,NA,temporal_context_id,02
+Test_2023_4,replicate observations,temporal_context,3,NA,temporal_context_id,03
+Test_2023_4,replicate observations,temporal_context,4,NA,temporal_context_id,04
+Test_2023_4,replicate observations,temporal_context,5,NA,temporal_context_id,05
Original file line number	Diff line number	Diff line change
Expand Up		@@ -28,5 +28,4 @@ Test_2023_4 is a copy of NHNSW_2023 with the following modifications:
		- Added `leaf_photosynthesis` and `leaf_stomatal_conductance` to test `repeat_measurements_id`, with another `leaf_stomatal_conductance_2` variable that doesn't specify `repeat_measurements_id`
		- Duplicated `fruit_length` for Imperata cylindrica and Alectryon subdentatus to test `collection_date` making separate `observation_id`'s and added `collection_date` column to dataset-level metadata


		See output/ for expected output files.