Skip to content

Commit

Permalink
Requested output file structure is now saved in the ConfigurationMana…
Browse files Browse the repository at this point in the history 
…ger.

Clean up of parsing code.

Workaround for ncType being non copiable.
  • Loading branch information
@donaldwj
donaldwj committed Mar 28, 2024
1 parent 7768fdf commit 69b042e
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 39 deletions.
21 changes: 15 additions & 6 deletions include/output/NetcdfOutputWriter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,22 @@ namespace data_output
struct NetcdfVariableDiscription
{
NetcdfVariableDiscription() {}
NetcdfVariableDiscription(const std::string& n, const NcType& t) : name(n), type(t), dim_names() {}
NetcdfVariableDiscription(const std::string& n, const NcType& t, const std::string& d_n ) : name(n), type(t), dim_names() { dim_names.push_back(d_n); }
NetcdfVariableDiscription(const std::string& n, const NcType& t, const std::vector<std::string>& d_n ) : name(n), type(t), dim_names(d_n) {}
NetcdfVariableDiscription(const std::string& n, const std::string& t) : name(n), type(t), dim_names() {}
NetcdfVariableDiscription(const std::string& n, const std::string& t, const std::string& d_n ) : name(n), type(t), dim_names() { dim_names.push_back(d_n); }
NetcdfVariableDiscription(const std::string& n, const std::string& t, const std::vector<std::string>& d_n ) : name(n), type(t), dim_names(d_n) {}

NetcdfVariableDiscription(const NetcdfVariableDiscription& src) : name(src.name), type(src.type), dim_names(src.dim_names) {}
NetcdfVariableDiscription& operator=(const NetcdfVariableDiscription& src)
{
name = src.name;
type = src.type;
dim_names = src.dim_names;
return *this;
}


std::string name;
NcType type;
std::string type;
std::vector<std::string> dim_names;
};

Expand Down Expand Up @@ -188,11 +197,11 @@ namespace data_output
var_dims.push_back(netcdfDims[name]);
}

netcdfVars[var.name] = netcdfFile->addVar(var.name, var.type, var_dims);
netcdfVars[var.name] = netcdfFile->addVar(var.name, data_output::strtonctype(var.type), var_dims);
}
else
{
netcdfVars[var.name] = netcdfFile->addVar(var.name, var.type);
netcdfVars[var.name] = netcdfFile->addVar(var.name, data_output::strtonctype(var.type));
}
}
}
Expand Down
39 changes: 39 additions & 0 deletions include/output/OutputFileDescription.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#ifndef __OUTPUT_FILE_DESCRIPTION_H__

#include <string>
#include <vector>

#include "output/NetcdfOutputWriter.hpp"

namespace data_output
{
enum FileOutputType { NetCDF4 = 1, CSV =2/*,JSON=3*/};

struct NetCDFFileInfo
{
public:

std::vector<data_output::NetcdfDimensionDiscription> dimensions;
std::vector<data_output::NetcdfVariableDiscription> variables;
};

struct CSVFileInfo
{
public:

std::vector<std::string> column_names;
std::vector<std::string> column_types;
};

struct OutputFileDescription
{
public:

std::string file_name;
FileOutputType file_type;
NetCDFFileInfo nc_info;
CSVFileInfo csv_info;
};
}

#endif // __OUTPUT_FILE_DESCRIPTION_H__
91 changes: 71 additions & 20 deletions include/realizations/catchment/Formulation_Manager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/algorithm/string.hpp>
#include <FeatureBuilder.hpp>
#include "JSONProperty.hpp"
#include "features/Features.hpp"
Expand All @@ -20,6 +21,7 @@
#include "routing/Routing_Params.h"
#include "LayerData.hpp"
#include "output/NetcdfOutputWriter.hpp"
#include "output/OutputFileDescription.hpp"

namespace realization {

Expand Down Expand Up @@ -192,41 +194,88 @@ namespace realization {
auto outputs_container = config_ptree.get_child_optional("outputs");

//Check to see if custom outputs have been defined
if (!outputs_container)
if (outputs_container)
{
for (std::pair<std::string, boost::property_tree::ptree> output_pair : *outputs_container)
for (std::pair<std::string, boost::property_tree::ptree> outputs_pair : *outputs_container)
{
auto& output_json_array = output_pair.second;
std::vector<data_output::NetcdfDimensionDiscription> output_dimensions;
std::vector<data_output::NetcdfVariableDiscription> output_variables;
std::vector< std::string > dimension_names;
data_output::FileOutputType type;
std::string output_file_path;

auto& output_json_array = outputs_pair.second;
for (std::pair<std::string, boost::property_tree::ptree> output_pair : output_json_array)
{
if (boost::to_lower_copy(output_pair.first) == "dim" )
if ( output_pair.first == "name" )
{
output_file_path = output_pair.second.get<std::string>("");
}
else if (output_pair.first == "type")
{
data_output::NetcdfDimensionDiscription nc_dim_des(
output_pair.second.get<std::string>("name"),
output_pair.second.get<int>("size"));
std::string string_type = output_pair.second.get<std::string>("");
if (string_type == "NetCDF4" )
{
type = data_output::NetCDF4;
}
else if (string_type == "CSV" )
{

}
}
else if (boost::to_lower_copy(output_pair.first) == "var" )
else if (boost::to_lower_copy(output_pair.first) == "dimensions" )
{
std::vector< std::string > dimension_names;
for( const std::pair<std::string, boost::property_tree::ptree>& dimension_pair : output_pair.second )
{
data_output::NetcdfDimensionDiscription nc_dim_des(
dimension_pair.first,
dimension_pair.second.get<int>("size"));

auto dimension_array = output_pair.second.get_child_optional("dimension");
if ( dimension_array )
output_dimensions.push_back(nc_dim_des);
dimension_names.push_back(dimension_pair.first);
}
}
else if (boost::to_lower_copy(output_pair.first) == "variables" )
{
auto variables_array = output_pair.second.get_child_optional("dimension");
if ( variables_array )
{
for(std::pair<std::string, boost::property_tree::ptree> dim_name_pair : *dimension_array )
for ( const std::pair<std::string, boost::property_tree::ptree>& variable_pair : *variables_array )
{
dimension_names.push_back(dim_name_pair.first);
}
std::string dimensions_str = variable_pair.second.get<std::string>("variables");
std::vector<std::string> dimensions_list;

data_output::NetcdfVariableDiscription nc_var_des(
variable_pair.first,
variable_pair.second.get<std::string>("type"),
dimensions_list);

output_variables.push_back(nc_var_des);
}
}

data_output::NetcdfVariableDiscription nc_var_des(
output_pair.second.get<std::string>("name"),
data_output::strtonctype(output_pair.second.get<std::string>("type")),
dimension_names);
}

output_files.push_back(data_output::OutputFileDescription());
output_files.back().file_name = output_file_path;
output_files.back().file_type = type;
switch ( type )
{
case data_output::NetCDF4:

output_files.back().nc_info.dimensions = output_dimensions;
output_files.back().nc_info.variables = output_variables;

break;

case data_output::CSV:

// TODO now use the variables and dimensions to make the file
;

break;
}
}


}

}
Expand Down Expand Up @@ -804,6 +853,8 @@ namespace realization {
bool using_routing = false;

ngen::LayerDataStorage layer_storage;

std::vector<data_output::OutputFileDescription> output_files;
};
}
#endif // NGEN_FORMULATION_MANAGER_H
2 changes: 1 addition & 1 deletion test/netcdf/Netcdf_Output_Integration_Test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class NetcdfOutputParserTest : public ::testing::Test {
{
nexus_data_path = find_file(path_options, "./data/nexus_data.geojson");
catchment_data_path = find_file(path_options, "./data/catchment_data.geojson");
realization_config_path = find_file(path_options, "./data/example_multilayer_realization_config.json");
realization_config_path = find_file(path_options, "./data/example_ncout_realization_config.json");
}

~NetcdfOutputParserTest() override {
Expand Down
23 changes: 11 additions & 12 deletions test/netcdf/Netcdf_Output_Test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ TEST_F(NetcdfOuputTest, TestNetcdfCreation) {
dimension_discription.push_back(data_output::NetcdfDimensionDiscription("lon",100));
dimension_discription.push_back(data_output::NetcdfDimensionDiscription("cat-id",1000));

variable_discription.push_back(data_output::NetcdfVariableDiscription("Ngen-Framework-Version", NC_STRING));
variable_discription.push_back(data_output::NetcdfVariableDiscription("lat", NC_FLOAT,"lat"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("lon", NC_FLOAT, "lon"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("RAINRATE", NC_FLOAT, std::vector<std::string>{"time","lat","lon"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("discharge", NC_FLOAT, std::vector<std::string>{"time","cat-id"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("Ngen-Framework-Version", "string"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("lat", "float","lat"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("lon", "float", "lon"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("RAINRATE", "float", std::vector<std::string>{"time","lat","lon"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("discharge", "float", std::vector<std::string>{"time","cat-id"}));

// make the output file
NetcdfOutputWriter output_file("netcdf-output-test.nc",dimension_discription,variable_discription);
Expand Down Expand Up @@ -104,11 +104,11 @@ TEST_F(NetcdfOuputTest, TestNetcdfWrite) {
dimension_discription.push_back(data_output::NetcdfDimensionDiscription("y",100));
dimension_discription.push_back(data_output::NetcdfDimensionDiscription("id",1000));

variable_discription.push_back(data_output::NetcdfVariableDiscription("version", NC_STRING));
variable_discription.push_back(data_output::NetcdfVariableDiscription("x", NC_FLOAT,"x"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("y", NC_FLOAT, "y"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("output1", NC_FLOAT, std::vector<std::string>{"time","x","y"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("output2", NC_FLOAT, std::vector<std::string>{"time","id"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("version", "string"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("x", "float:","x"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("y", "float", "y"));
variable_discription.push_back(data_output::NetcdfVariableDiscription("output1", "float", std::vector<std::string>{"time","x","y"}));
variable_discription.push_back(data_output::NetcdfVariableDiscription("output2", "float", std::vector<std::string>{"time","id"}));

NetcdfOutputWriter output_file("netcdf-write-test-1.nc",dimension_discription, variable_discription);

Expand All @@ -119,8 +119,7 @@ TEST_F(NetcdfOuputTest, TestNetcdfWrite) {
data_vec[i] = i * 0.1f;
}

auto s = output_file["output2"];
s << nc_offset(0,0) << nc_stride(1,1000) << data_vec;
output_file["output2"] << nc_offset(0,0) << nc_stride(1,1000) << data_vec;

SUCCEED();
}
Expand Down

0 comments on commit 69b042e

Please sign in to comment.