From 83ad32347d234b7e47214ac41408480623e5d84d Mon Sep 17 00:00:00 2001 From: Dylan Beaudette Date: Tue, 15 Feb 2022 11:02:02 -0800 Subject: [PATCH] addressing #25, fulltext section data for SoilWeb now successfully created --- misc/parseOSD-dev/create-SoilWeb-OSD-data.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc/parseOSD-dev/create-SoilWeb-OSD-data.R b/misc/parseOSD-dev/create-SoilWeb-OSD-data.R index cd8a71879c..b7232a0f31 100644 --- a/misc/parseOSD-dev/create-SoilWeb-OSD-data.R +++ b/misc/parseOSD-dev/create-SoilWeb-OSD-data.R @@ -21,7 +21,7 @@ sc <- sc$soilseriesname ## TODO: double-check funky names like "O'BRIEN" and chars not [a-z] -## TODO: add sections / entire OSD -> fulltext data +## TODO: add entire OSD -> fulltext data ## TODO: narratives in the JSON files have leading white space @@ -160,8 +160,10 @@ write.csv(hz, file = gzfile('parsed-data.csv.gz'), row.names = FALSE) write.csv(s, file = gzfile('parsed-site-data.csv.gz'), row.names = FALSE) ## re-make section fulltext table + INSERT statements +# 6 minutes system.time(.makeFullTextSectionsTable(fulltext.records)) - +# gzip +R.utils::gzip('fulltext-section-data.sql', overwrite = TRUE) ## TODO: investigate missing records, relative to the last time this was run # nrow(read.csv('E:/working_copies/parse-osd/R/parsed-data.csv.gz'))