feature: add parsing of task_id, test_code and user_output to bring t…

…est-runner to v3 (#72)

* Update Catch2 version to introduce tags as task_id

* Update to alpine to use c++ formatter

* Add code for cpp xml2json parser

* Add test for task_id in test case

* Update expected results for new catch2, alpine and v3 version

* Add new parser compilation

* Change parsing to new cpp version

* Remove Python parsing content

* Add test case for user output

* Add json parsing of cout user output

* Add auto conversion to work with Catch2 v2 and v3

* Add test case for catch2 version conversion

* Add test_case data generation

* Update call and compilation of the parser

* Update expected results to include test_case data

* Add test case for test_case data parsing

Dockerfile

@@ -1,13 +1,11 @@
-FROM alpine:3.11
+FROM alpine:3.18
 
-RUN apk add --no-cache coreutils g++ libc-dev cmake make python3 git boost-dev
-RUN pip3 install --upgrade pip
-RUN pip3 install junitparser
+RUN apk add --no-cache coreutils g++ libc-dev cmake make git boost-dev
 
 # Build Catch as a library directly inside the Docker image
 # since it takes a bit of time to compile it,this way we speed
 # compilation of tests when running the test suite.
-RUN git clone https://github.com/catchorg/Catch2.git --depth 1 --branch v2.13.6 && \
+RUN git clone https://github.com/catchorg/Catch2.git --depth 1 --branch v3.4.0 && \
     cd Catch2 && \
     cmake -Bbuild -H. -DCMAKE_CXX_STANDARD=17 -DCATCH_BUILD_STATIC_LIBRARY=ON -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF && \
     cmake --build build/ --target install && \
@@ -16,4 +14,5 @@ RUN git clone https://github.com/catchorg/Catch2.git --depth 1 --branch v2.13.6
 
 WORKDIR /opt/test-runner
 COPY . .
+RUN g++ -I ./include ./src/*.cpp -o ./bin/exercism_parser -lboost_system -lboost_json
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

bin/run.sh

@@ -25,11 +25,11 @@ slug="$1"
 input_dir="${2%/}"
 output_dir="${3%/}"
 build_dir="/tmp/${slug}"
-process_file="/opt/test-runner/bin/process.py"
 compilation_errors_file_name="compilation-errors"
 test_output_file_name="test-output.xml"
 results_file="${output_dir}/results.json"
 binary_file="${build_dir}/${slug}"
+test_file_path="${build_dir}/${slug//-/_}_test.cpp"
 
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
@@ -40,13 +40,15 @@ echo "${slug}: testing..."
 # the makefile uses the directory name to determine the files
 cp -R "${input_dir}/" "${build_dir}" && cd "${build_dir}"
 
+# Replace the old Catch2 v2 include line with Catch2 v3
+sed -i -e 's/#include <catch2\/catch.hpp>/#include <catch2\/catch_all.hpp>/g' "${test_file_path}"
+
 cmake -DEXERCISM_TEST_SUITE=1 -DEXERCISM_RUN_ALL_TESTS=1 .
 make 2> "${compilation_errors_file_name}"
 
 # In case of compilation errors the executable will not be created
-[[ -f "./${slug}" ]] && chmod +x "./${slug}" && "./${slug}" -r junit -o "${test_output_file_name}"
-
-python3 "${process_file}" "${build_dir}/${compilation_errors_file_name}" "${build_dir}/${test_output_file_name}" "${results_file}"
+[[ -f "./${slug}" ]] && chmod +x "./${slug}" && "./${slug}" -r xml -o "${test_output_file_name}"
+/opt/test-runner/bin/exercism_parser "${build_dir}/${test_output_file_name}" "${results_file}" "${build_dir}/${compilation_errors_file_name}" "${test_file_path}"
 
 cd -
 

include/helpers.h

@@ -0,0 +1,72 @@
+#include <algorithm>
+#include <functional>
+#include <cctype>
+#include <locale>
+
+
+// trimming function taken from: https://stackoverflow.com/a/217605/4919081
+
+// trim from start (in place)
+static inline void ltrim(std::string &s)
+{
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+                                    std::not1(std::ptr_fun<int, int>(std::isspace))));
+}
+
+// trim from end (in place)
+static inline void rtrim(std::string &s)
+{
+    s.erase(std::find_if(s.rbegin(), s.rend(),
+                         std::not1(std::ptr_fun<int, int>(std::isspace)))
+                .base(),
+            s.end());
+}
+
+// trim from both ends (in place)
+static inline void trim(std::string &s)
+{
+    rtrim(s);
+    ltrim(s);
+}
+
+// trim from start (copying)
+static inline std::string ltrim_copy(std::string s)
+{
+    ltrim(s);
+    return s;
+}
+
+// trim from end (copying)
+static inline std::string rtrim_copy(std::string s)
+{
+    rtrim(s);
+    return s;
+}
+
+// trim from both ends (copying)
+static inline std::string trim_copy(std::string s)
+{
+    trim(s);
+    return s;
+}
+
+static inline std::string read_file(std::string_view path)
+{
+    constexpr auto read_size = std::size_t(4096);
+    auto stream = std::ifstream(path.data());
+    stream.exceptions(std::ios_base::badbit);
+
+    if (not stream)
+    {
+        return "compilation error file does not exist";
+    }
+
+    auto out = std::string();
+    auto buf = std::string(read_size, '\0');
+    while (stream.read(&buf[0], read_size))
+    {
+        out.append(buf, 0, stream.gcount());
+    }
+    out.append(buf, 0, stream.gcount());
+    return out;
+}

include/xml2json.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <boost/property_tree/ptree.hpp>
+
+#include <string>
+#include <vector>
+
+namespace pt = boost ::property_tree;
+
+struct Test_result
+{
+    std::string name{};
+    std::string status{};
+    std::string message{};
+    std::string output{};
+    std::string test_code{};
+    int task_id{};
+    void add_result_to_json(std::ofstream &json_file) const;
+};
+
+struct Output_message
+{
+    int version{3};
+    std::string status{};
+    std::string message{};
+    std::vector<Test_result> tests{};
+    void load_from_catch2_xml(const std::string &xml_test_output_file, const std::string &compilation_error_file);
+    void save_as_exercism_json(const std::string &filename);
+    std::string build_test_message(const pt::ptree &tree);
+    void generate_test_code_from_test_file(const std::string &test_file_path);
+};

src/exercism_parser.cpp

@@ -0,0 +1,30 @@
+#include <iostream>
+#include "xml2json.h"
+
+int main(int argc, char *argv[])
+{
+    if (argc < 5)
+    {
+        std::cout << "Input, output, compilation errors, and test file names are required but were not supplied.\n"
+                  << "Usage: " << argv[0] << " input_file_name output_file_name compilation_errors_file_name test_file_path\n";
+        return -1;
+    }
+
+    std::string input_file_file_name_path{argv[1]};
+    std::string output_file_file_name_path{argv[2]};
+    std::string compilation_errors_file_name_path{argv[3]};
+    std::string test_file_path{argv[4]};
+
+    try
+    {
+        Output_message om{};
+        om.load_from_catch2_xml(input_file_file_name_path, compilation_errors_file_name_path);
+        om.generate_test_code_from_test_file(test_file_path);
+        om.save_as_exercism_json(output_file_file_name_path);
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Error: " << e.what() << "\n";
+    }
+    return 0;
+}