Merge pull request #38 from ecmwf/feature/github_actions

adding github actions for downstream ci

.github/ci-config.yml

@@ -0,0 +1,5 @@
+cmake_options: -DBOOST_ROOT=/usr -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 -DENABLE_STATIC_BOOST_LIBS=OFF
+dependencies: |
+  ecmwf/ecbuild
+dependency_branch: develop
+parallelism_factor: 8

.github/ci-hpc-config.yml

@@ -0,0 +1,9 @@
+build:
+  modules:
+    - boost
+    - ninja
+    - python3
+    - qt
+  dependencies:
+    - ecmwf/ecbuild@develop
+  parallel: 64

.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'master'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      ecflow: ecmwf/ecflow@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+  # Run CI of private downstream packages on self-hosted runners
+  private-downstream-ci:
+    name: private-downstream-ci
+    needs: [downstream-ci]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci
+          payload: '{"ecflow": "ecmwf/ecflow@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      ecflow: ecmwf/ecflow@${{ github.event.pull_request.head.sha || github.sha }}
+      skip_matrix_jobs: intel-2021.4.0
+    secrets: inherit
+
+  # Run CI of private downstream packages on HPC
+  private-downstream-ci-hpc:
+    name: private-downstream-ci-hpc
+    needs: [downstream-ci-hpc]
+    if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Dispatch private downstream CI
+        uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+        with:
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          owner: ecmwf-actions
+          repository: private-downstream-ci
+          event_type: downstream-ci-hpc
+          payload: '{"ecflow": "ecmwf/ecflow@${{ github.event.pull_request.head.sha || github.sha }}"}'

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

Server/CMakeLists.txt

@@ -94,15 +94,20 @@ target_clangformat(ecflow_server)
 # ================================================================================                         
 # TEST
 # libboost_unit_test_framework  undefined reference to `clock_gettime', ${LIBRT} needed for boost 1.71
-ecbuild_add_test( TARGET       u_server
-                  SOURCES      test/TestServerEnvironment.cpp 
-                               test/TestServer1.cpp
-                  INCLUDES     src ${Boost_INCLUDE_DIRS}
-                  LIBS         libserver ${OPENSSL_LIBRARIES}
-                               ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${Boost_TEST_EXEC_MONITOR_LIBRARY} ${LIBRT}
-                  DEFINITIONS  ${BOOST_TEST_DYN_LINK}
-                  TEST_DEPENDS u_base
-              )
+ecbuild_add_test(
+  TARGET
+    u_server
+  INCLUDES
+    src
+  SOURCES
+    test/TestServerEnvironment.cpp
+    test/TestServer1.cpp
+  LIBS
+    libserver
+    $<$<BOOL:${OPENSSL_FOUND}>:OpenSSL::SSL>
+  TEST_DEPENDS
+    u_base
+)
 
 target_clangformat(u_server CONDITION ENABLE_TESTS)
 

Server/test/TestServerEnvironment.cpp

@@ -20,7 +20,7 @@
 
 #include <boost/filesystem/operations.hpp>
 #include <boost/lexical_cast.hpp>
-#include <boost/test/unit_test.hpp>
+#include <boost/test/included/unit_test.hpp>
 
 #include "CheckPt.hpp"
 #include "Ecf.hpp"

Udp/test/TestSupport.hpp

@@ -21,10 +21,12 @@
 
 #include "ClientInvoker.hpp"
 #include "Defs.hpp"
+#include "EcfPortLock.hpp"
 #include "File.hpp"
 #include "Host.hpp"
 #include "Node.hpp"
 #include "NodeAttr.hpp"
+#include "Str.hpp"
 #include "UDPClient.hpp"
 
 namespace bp = boost::process;
@@ -74,7 +76,7 @@ class MockServer : public BaseMockServer<MockServer> {
     explicit MockServer(port_t port) : BaseMockServer<MockServer>(ecf::Host{}.name(), port) {}
 
     void load_definition(const std::string& defs) const {
-        ClientInvoker client(host(), port());
+        ClientInvoker client(ecf::Str::LOCALHOST(), port());
         auto error = client.loadDefs(defs);
         BOOST_REQUIRE_MESSAGE(!error, "unable to load definitions");
         std::cout << "   MOCK: reference ecFlow suite has been loaded" << std::endl;
@@ -97,7 +99,8 @@ class MockServer : public BaseMockServer<MockServer> {
 
 private:
     node_ptr get_node_at(const std::string& path) const {
-        ClientInvoker client(host(), port());
+        std::cout << "   Creating ecflow_client connected to " << host() << ":" << port() << std::endl;
+        ClientInvoker client(ecf::Str::LOCALHOST(), port());
 
         // load all definitions
         std::shared_ptr<Defs> defs = nullptr;
@@ -125,15 +128,22 @@ class MockServer : public BaseMockServer<MockServer> {
         // Just for precaution, in case a previous run didn't clean up...
         cleanup(host, port);
 
-        std::string invoke_command = ecf::File::root_build_dir() + "/bin/ecflow_server";
+        std::string invoke_command = ecf::File::find_ecf_server_path();
+
+        BOOST_REQUIRE_MESSAGE(!invoke_command.empty(), "The server program could not be found");
+        BOOST_REQUIRE_MESSAGE(boost::filesystem::exists(invoke_command),
+                              "Server exe does not exist at:" << invoke_command);
+
         invoke_command += " --port ";
         invoke_command += std::to_string(port);
         invoke_command += " -d &";
 
+        std::cout << "Launching ecflow_server @" << host << ":" << port << ", with: " << invoke_command << std::endl;
+
         bp::child child(invoke_command);
 
-        ClientInvoker client(host, port);
-        if (!client.wait_for_server_reply(1)) {
+        ClientInvoker client(ecf::Str::LOCALHOST(), port);
+        if (!client.wait_for_server_reply(5)) {
             BOOST_REQUIRE_MESSAGE(false, "could not launch ecflow server");
         }
 
@@ -204,7 +214,9 @@ class MockUDPServer : public BaseMockServer<MockUDPServer> {
     }
 
     static void sendRequest(uint16_t port, const std::string& request) {
-        ecf::UDPClient client("localhost", std::to_string(port));
+        const std::string host = "localhost";
+        std::cout << "   Creating ecflow_udp client connected to " << host << ":" << port << std::endl;
+        ecf::UDPClient client(host, std::to_string(port));
         client.send(request);
 
         // Wait for request to flow...
@@ -223,6 +235,8 @@ class MockUDPServer : public BaseMockServer<MockUDPServer> {
         invoke_command += std::to_string(ecflow_port);
         invoke_command += " --verbose";
 
+        std::cout << "   Launching ecflow_udp @" << host << ":" << port << ", with: " << invoke_command << std::endl;
+
         bp::child server(invoke_command);
 
         // Wait for server to start...
@@ -241,14 +255,32 @@ class MockUDPServer : public BaseMockServer<MockUDPServer> {
  */
 struct EnableServersFixture
 {
-    EnableServersFixture() : ecflow_server(42424), ecflow_udp(42425, 42424) {
-        // Load 'reference' suite for tests...
-        ecflow_server.load_definition("data/reference.def");
-    }
+    EnableServersFixture() : EnableServersFixture(get_ecflow_server_port(), get_ecflow_udp_port()) {}
     ~EnableServersFixture() = default;
 
     ecf::test::MockServer ecflow_server;
     ecf::test::MockUDPServer ecflow_udp;
+
+private:
+    static MockServer::port_t get_ecflow_server_port() {
+        MockServer::port_t selected_port = 3199;
+        std::cout << "   Attempting to use port: " << selected_port << std::endl;
+        while (!EcfPortLock::is_free(selected_port)) {
+            std::cout << "   Selected port: " << selected_port << " is not available." << std::endl;
+            ++selected_port;
+            std::cout << "   Attempting to use port: " << selected_port << std::endl;
+        }
+        std::cout << "   Found free port: " << selected_port << "\n";
+        return selected_port;
+    }
+    static MockServer::port_t get_ecflow_udp_port() { return 3198; }
+
+    EnableServersFixture(MockServer::port_t ecflow_server_port, MockServer::port_t ecflow_udp_port)
+        : ecflow_server(ecflow_server_port),
+          ecflow_udp(ecflow_udp_port, ecflow_server_port) {
+        // Load 'reference' suite for tests...
+        ecflow_server.load_definition("data/reference.def");
+    }
 };
 
 } // namespace ecf::test