Skip to content

Commit

Permalink
server: resolve ECF_INCLUDE when preprocessing .ecf files
Browse files Browse the repository at this point in the history 
Re ECFLOW-1920
  • Loading branch information
@marcosbento
marcosbento committed Oct 26, 2023
1 parent 7a96afc commit 3f3893c
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 37 deletions.
86 changes: 51 additions & 35 deletions ANode/src/EcfFile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,48 @@ void EcfFile::extract_used_variables(NameValueMap& used_variables_as_map,
}
}

/**
* Retrieve the set of paths defined by ECF_INCLUDE.
*
* ECF_INCLUDE is either a single path, or a list of paths (separated by ':').
* The content of ECF_INCLUDE can reference ecFlow variables, which are replaced in the result.
*
* @param ecf
* The ECF file to consider (the Node directly related with the file determines which ECF_INCLUDE value is used)
*
* @return
* An empty set if ECF_INCLUDE is not defined; otherwise, the set of paths defined by ECF_INCLUDE, after resolving
* all ecFlow variables.
*/
std::vector<std::string> EcfFile::get_ecf_include_paths(const EcfFile& ecf) {
assert(ecf.node_);
const Node& node = *ecf.node_;

std::string ecf_include;
node.findParentUserVariableValue(Str::ECF_INCLUDE(), ecf_include);

std::vector<std::string> paths;
if (!ecf_include.empty()) {

// if ECF_INCLUDE is a set a paths, search in order. i.e., like $PATH
if (ecf_include.find(':') != std::string::npos) {
Str::split(ecf_include, paths, ":");
}
else {
paths = {ecf_include};
}

// Don't rely on hard coded paths. Added for testing, but could be generally useful
// since in test scenario ECF_INCLUDE is defined relative to $ECF_HOME
for (std::string& path : paths) {
node.variable_dollar_subsitution(path);
node.variableSubsitution(path);
}
}

return paths;
}

bool EcfFile::open_script_file(const std::string& file_or_cmd,
EcfFile::Type type,
std::vector<std::string>& lines,
Expand Down Expand Up @@ -1690,45 +1732,19 @@ std::string PreProcessor::getIncludedFilePath(const std::string& includedFile1,

Node* node = ecfile_->node_;
if (includedFile[0] == '<') {
// %include <filename> can be one of:
// o When ECF_INCLUDE is a single path -> path1/filename
// o When ECF_INCLUDE is a multi path -> path1:path2:path3 -> ECFLOW-261
// -> path1/filename || path2/filename || path3/filename
// o ECF_HOME/filename
std::string ecf_include;
if (node->findParentUserVariableValue(Str::ECF_INCLUDE(), ecf_include) && !ecf_include.empty()) {

// if ECF_INCLUDE is a set a paths, search in order. i.e like $PATH
if (ecf_include.find(':') != std::string::npos) {
std::vector<std::string> include_paths;
Str::split(ecf_include, include_paths, ":");
for (const auto& include_path : include_paths) {
ecf_include.clear();
ecf_include = include_path;
ecf_include += '/';
ecf_include += the_include_file;

// Don't rely on hard coded paths. Added for testing, but could be generally useful
// since in test scenario ECF_INCLUDE is defined relative to $ECF_HOME
node->variable_dollar_subsitution(ecf_include);

if (ecfile_->file_exists(ecf_include))
return ecf_include;
}
}
else {
ecf_include += '/';
ecf_include += the_include_file;
node->variable_dollar_subsitution(ecf_include);
if (ecfile_->file_exists(ecf_include))
return ecf_include;
}

// ECF_INCLUDE is specified *BUT* the file does *NOT* exist, Look in ECF_HOME
auto ecf_include_paths = EcfFile::get_ecf_include_paths(*ecfile_);
for (const auto& ecf_include_path : ecf_include_paths) {
auto include_path_plus_file = ecf_include_path;
include_path_plus_file += '/';
include_path_plus_file += the_include_file;

if (ecfile_->file_exists(include_path_plus_file))
return include_path_plus_file;
}

// WE get HERE *if* ECF_INCLUDE not specified, or if specified but file *not found*
ecf_include.clear();
std::string ecf_include;
node->findParentVariableValue(Str::ECF_HOME(), ecf_include);
if (ecf_include.empty()) {
std::stringstream ss;
Expand Down
2 changes: 2 additions & 0 deletions ANode/src/EcfFile.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ class EcfFile {
static void extract_used_variables(NameValueMap& used_variables_as_map,
const std::vector<std::string>& script_lines);

static std::vector<std::string> get_ecf_include_paths(const EcfFile& ecf);

private:
friend class PreProcessor;
enum Type { SCRIPT, INCLUDE, MANUAL, COMMENT };
Expand Down
2 changes: 1 addition & 1 deletion ANode/src/Node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1452,7 +1452,7 @@ bool Node::find_all_used_variables(std::string& cmd, NameValueMap& used_variable
return true;
}

bool Node::variable_dollar_subsitution(std::string& cmd) {
bool Node::variable_dollar_subsitution(std::string& cmd) const {
// scan command for environment variables, and substitute
// edit ECF_INCLUDE $ECF_HOME/include

Expand Down
2 changes: 1 addition & 1 deletion ANode/src/Node.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ class Node : public std::enable_shared_from_this<Node> {
/// Find all environment variables, in the input string and substitute.
/// with correspondingly named variable value.
/// i.e search for ${ENV} and replace
bool variable_dollar_subsitution(std::string& cmd);
bool variable_dollar_subsitution(std::string& cmd) const;

/// Resolve inlimit references to limits, and check trigger and complete expression
virtual bool check(std::string& errorMsg, std::string& warningMsg) const;
Expand Down
46 changes: 46 additions & 0 deletions ANode/test/TestEcfFile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1650,4 +1650,50 @@ BOOST_AUTO_TEST_CASE(test_ecf_micro_with_comments_ECFLOW_1686) {
basic_test_template("test_ecf_micro_with_comments_ECFLOW_1686", ecf_file, expected_job_file_contents);
}

BOOST_AUTO_TEST_CASE(test_ecf_file_resolve_single_ecf_include_with_dollar) {
Defs d;
suite_ptr s = d.add_suite("suite"); // suite suite
s->add_variable("CORE", "/path/to/core"); // edit CORE /path/to/core
s->add_variable("ECF_INCLUDE", "$CORE/%SUITE%/include"); // edit ECF_INCLUDE $CORE/include
task_ptr t1 = s->add_task("t1"); // task t1

std::string ecf_file_location;
EcfFile ecf(t1.get(), ecf_file_location);

auto ecf_include_paths = EcfFile::get_ecf_include_paths(ecf);
BOOST_REQUIRE_EQUAL(ecf_include_paths.size(), 1);
BOOST_REQUIRE_EQUAL(ecf_include_paths[0], "/path/to/core/suite/include");
}

BOOST_AUTO_TEST_CASE(test_ecf_file_resolve_single_ecf_include) {
Defs d;
suite_ptr s = d.add_suite("suite"); // suite suite
s->add_variable("CORE", "/path/to/core"); // edit CORE /path/to/core
s->add_variable("ECF_INCLUDE", "%CORE%/%SUITE%/include"); // edit ECF_INCLUDE %CORE%/%SUITE%/include
task_ptr t1 = s->add_task("t1"); // task t1

std::string ecf_file_location;
EcfFile ecf(t1.get(), ecf_file_location);

auto ecf_include_paths = EcfFile::get_ecf_include_paths(ecf);
BOOST_REQUIRE_EQUAL(ecf_include_paths.size(), 1);
BOOST_REQUIRE_EQUAL(ecf_include_paths[0], "/path/to/core/suite/include");
}

BOOST_AUTO_TEST_CASE(test_ecf_file_resolve_multiple_ecf_include) {
Defs d;
suite_ptr s = d.add_suite("suite"); // suite suite
s->add_variable("CORE", "/path/to/core"); // edit CORE /path/to/core
s->add_variable("ECF_INCLUDE", "%CORE%/%SUITE%/include:%CORE%/more"); // edit ECF_INCLUDE %CORE%/%SUITE%...
task_ptr t1 = s->add_task("t1"); // task t1

std::string location;
EcfFile ecf(t1.get(), location);

auto ecf_include_paths = EcfFile::get_ecf_include_paths(ecf);
BOOST_REQUIRE_EQUAL(ecf_include_paths.size(), 2);
BOOST_REQUIRE_EQUAL(ecf_include_paths[0], "/path/to/core/suite/include");
BOOST_REQUIRE_EQUAL(ecf_include_paths[1], "/path/to/core/more");
}

BOOST_AUTO_TEST_SUITE_END()

0 comments on commit 3f3893c

Please sign in to comment.