libfaasm: add support for general S3 API (#126)

* libfaasm: add support for general s3 API

* s3: add function to get number of buckets

* s3: add remaining basic functions

* nits: run clang-format

* gh: bump minor code version

* libfaasm: remove un-implemented symbols

* s3: temporarily commit the workflow functions

* wc: more functions

* s3: untrack wc functions

* s3: build functions as part of gha

* func(demo): update chain_output function for new signature

* func: separate threaded/non-threaded targets

.env

@@ -1,3 +1,3 @@
-SYSROOT_VERSION=0.5.0
-SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.5.0
+SYSROOT_VERSION=0.6.0
+SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.6.0
 COMPOSE_PROJECT_NAME=cpp-dev

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm.azurecr.io/cpp-sysroot:0.5.0
+      image: faasm.azurecr.io/cpp-sysroot:0.6.0
     steps:
       # --- Update code ---
       - name: "Checkout code"

VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

func/CMakeLists.txt

@@ -74,4 +74,5 @@ add_subdirectory(dynlink)
 add_subdirectory(errors)
 add_subdirectory(mpi)
 add_subdirectory(omp)
+add_subdirectory(s3)
 add_subdirectory(threads)

func/demo/chain_output.cpp

@@ -26,28 +26,37 @@ int main(int argc, char* argv[])
     unsigned int callIdB = faasmChain(otherB, nullptr, 0);
 
     std::string expectedA = "expected A";
-    std::string actualA;
-    actualA.reserve(expectedA.size());
 
     std::string expectedB = "longer expected B";
-    std::string actualB;
-    actualB.reserve(expectedB.size());
 
+    std::string actualA;
+    char* actualABuf;
+    int actualABufSize;
     unsigned int resA =
-      faasmAwaitCallOutput(callIdA, actualA.c_str(), actualA.size());
+      faasmAwaitCallOutput(callIdA, &actualABuf, &actualABufSize);
+    actualA.assign(actualABuf, actualABuf + actualABufSize);
+
+    std::string actualB;
+    char* actualBBuf;
+    int actualBBufSize;
     unsigned int resB =
-      faasmAwaitCallOutput(callIdB, actualB.c_str(), actualB.size());
+      faasmAwaitCallOutput(callIdB, &actualBBuf, &actualBBufSize);
+    actualB.assign(actualBBuf, actualBBuf + actualBBufSize);
 
     if (resA != 0 || resB != 0) {
         printf("One or more chained calls failed: %i %i\n", resA, resB);
         return 1;
     }
 
     if (actualA != expectedA) {
+        printf(
+          "Output mismatch: %s != %s\n", actualA.c_str(), expectedA.c_str());
         return 1;
     }
 
     if (actualB != expectedB) {
+        printf(
+          "Output mismatch: %s != %s\n", actualB.c_str(), expectedB.c_str());
         return 1;
     }
 

func/s3/CMakeLists.txt

@@ -0,0 +1,16 @@
+set(FAASM_USER s3)
+
+function(s3_func exec_name dir_path)
+    faasm_func(${exec_name} ${dir_path})
+    set(ALL_DEMO_FUNCS ${ALL_DEMO_FUNCS} ${exec_name} PARENT_SCOPE)
+endfunction(s3_func)
+
+s3_func(get_num_buckets get_num_buckets.cpp)
+s3_func(list_buckets list_buckets.cpp)
+s3_func(get_num_keys get_num_keys.cpp)
+s3_func(list_keys list_keys.cpp)
+s3_func(add_key_bytes add_key_bytes.cpp)
+s3_func(get_key_bytes get_key_bytes.cpp)
+
+# Custom target to group all the demo functions
+add_custom_target(s3_all_funcs DEPENDS ${ALL_DEMO_FUNCS})

func/s3/add_key_bytes.cpp

@@ -0,0 +1,30 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+
+int main(int argc, char* argv[])
+{
+    // Get bucket and key from command line
+    if (argc != 3) {
+        printf("error: must invoke function with two arguments: bucketName "
+               "keyName\n");
+        return 1;
+    }
+
+    char* bucketName = argv[1];
+    char* keyName = argv[2];
+
+    // Get the bytes to add as input
+    int inputSize = faasmGetInputSize();
+    uint8_t keyBytes[inputSize];
+    faasmGetInput(keyBytes, inputSize);
+
+    int ret =
+      __faasm_s3_add_key_bytes(bucketName, keyName, (void*)keyBytes, inputSize);
+
+    return ret;
+}

func/s3/get_key_bytes.cpp

@@ -0,0 +1,30 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+
+int main(int argc, char* argv[])
+{
+    // Get bucket and key from command line
+    if (argc != 3) {
+        printf("error: must invoke function with two arguments: bucketName "
+               "keyName\n");
+        return 1;
+    }
+
+    char* bucketName = argv[1];
+    char* keyName = argv[2];
+
+    uint8_t* keyBytes;
+    int keyBytesLen;
+
+    int ret =
+      __faasm_s3_get_key_bytes(bucketName, keyName, &keyBytes, &keyBytesLen);
+    printf("Got %s/%s: %s\n", bucketName, keyName, (char*)keyBytes);
+    faasmSetOutput((char*)keyBytes, keyBytesLen);
+
+    return ret;
+}

func/s3/get_num_buckets.cpp

@@ -0,0 +1,20 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string>
+
+int main(int argc, char* argv[])
+{
+    int numBuckets = __faasm_s3_get_num_buckets();
+
+    printf("Got %i buckets!\n", numBuckets);
+
+    std::string numBucketsStr = std::to_string(numBuckets);
+    faasmSetOutput(numBucketsStr.c_str(), numBucketsStr.size());
+
+    return 0;
+}

func/s3/get_num_keys.cpp

@@ -0,0 +1,25 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string>
+
+int main(int argc, char* argv[])
+{
+    // Get the bucket name as an input
+    int inputSize = faasmGetInputSize();
+    char bucketName[inputSize];
+    faasmGetInput((uint8_t*)bucketName, inputSize);
+
+    int numKeys = __faasm_s3_get_num_keys(bucketName);
+
+    printf("Bucket %s has %i keys!\n", bucketName, numKeys);
+
+    std::string numKeysStr = std::to_string(numKeys);
+    faasmSetOutput(numKeysStr.c_str(), numKeysStr.size());
+
+    return 0;
+}

func/s3/list_buckets.cpp

@@ -0,0 +1,42 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char* argv[])
+{
+    int numBuckets = __faasm_s3_get_num_buckets();
+
+    char* bucketsBuffer[numBuckets];
+    int bucketsBufferLens[numBuckets];
+    __faasm_s3_list_buckets(bucketsBuffer, bucketsBufferLens);
+
+    int totalSize = 0;
+    for (int i = 0; i < numBuckets; i++) {
+        totalSize += bucketsBufferLens[i];
+    }
+    totalSize += numBuckets - 1;
+
+    // Prepare the output: instead of a newline use a '|' character
+    char outBuffer[totalSize];
+
+    printf("Got %i buckets!\n", numBuckets);
+    int offset = 0;
+    for (int i = 0; i < numBuckets; i++) {
+        strncpy(outBuffer + offset, bucketsBuffer[i], bucketsBufferLens[i]);
+        offset += bucketsBufferLens[i];
+        if (i < numBuckets - 1) {
+            outBuffer[offset] = (char)'|';
+            offset += 1;
+        }
+        printf("Bucket %i: %s\n", i, bucketsBuffer[i]);
+    }
+
+    faasmSetOutput(outBuffer, totalSize);
+
+    return 0;
+}

func/s3/list_keys.cpp

@@ -0,0 +1,47 @@
+extern "C"
+{
+#include "faasm/host_interface.h"
+}
+
+#include <faasm/faasm.h>
+#include <stdio.h>
+#include <string.h>
+
+int main(int argc, char* argv[])
+{
+    // Get the bucket name as an input
+    int inputSize = faasmGetInputSize();
+    char bucketName[inputSize];
+    faasmGetInput((uint8_t*)bucketName, inputSize);
+
+    int numKeys = __faasm_s3_get_num_keys(bucketName);
+
+    char* keysBuffer[numKeys];
+    int keysBufferLens[numKeys];
+    __faasm_s3_list_keys(bucketName, keysBuffer, keysBufferLens);
+
+    int totalSize = 0;
+    for (int i = 0; i < numKeys; i++) {
+        totalSize += keysBufferLens[i];
+    }
+    totalSize += numKeys - 1;
+
+    // Prepare the output: instead of a newline use a '|' character
+    char outBuffer[totalSize];
+
+    printf("Bucket %s has %i keys!\n", bucketName, numKeys);
+    int offset = 0;
+    for (int i = 0; i < numKeys; i++) {
+        strncpy(outBuffer + offset, keysBuffer[i], keysBufferLens[i]);
+        offset += keysBufferLens[i];
+        if (i < numKeys - 1) {
+            outBuffer[offset] = (char)'|';
+            offset += 1;
+        }
+        printf("Key %i: %s\n", i, keysBuffer[i]);
+    }
+
+    faasmSetOutput(outBuffer, totalSize);
+
+    return 0;
+}

libfaasm/core.cpp

@@ -152,8 +152,8 @@ unsigned int faasmAwaitCall(unsigned int messageId)
 }
 
 unsigned int faasmAwaitCallOutput(unsigned int messageId,
-                                  const char* output,
-                                  long outputLen)
+                                  char** output,
+                                  int* outputLen)
 {
     return __faasm_await_call_output(messageId, output, outputLen);
 }

libfaasm/faasm/core.h

@@ -188,8 +188,8 @@ extern "C"
      * Gets the output from the given call into the buffer
      */
     unsigned int faasmAwaitCallOutput(unsigned int messageId,
-                                      const char* output,
-                                      long outputLen);
+                                      char** output,
+                                      int* outputLen);
 
     /**
      * Returns the python user

libfaasm/faasm/host_interface.h

@@ -114,8 +114,8 @@ int __faasm_await_call(unsigned int messageId);
 
 HOST_IFACE_FUNC
 int __faasm_await_call_output(unsigned int messageId,
-                              const char* output,
-                              long outputLen);
+                              char** output,
+                              int* outputLen);
 
 HOST_IFACE_FUNC
 void __faasm_get_py_user(unsigned char* buffer, long bufferLen);
@@ -160,4 +160,35 @@ void __faasm_migrate_point(FaasmMigrateEntryPoint f, int arg);
 
 HOST_IFACE_FUNC
 void __faasm_host_interface_test(int testNum);
+
+// ----- S3 -----
+
+HOST_IFACE_FUNC
+int __faasm_s3_get_num_buckets();
+
+// Note that bucketsBuffer is, in reality, a char** populated by the host
+HOST_IFACE_FUNC
+void __faasm_s3_list_buckets(void* bucketsBuffer, int* bucketsBufferLens);
+
+HOST_IFACE_FUNC
+int __faasm_s3_get_num_keys(const char* bucketName);
+
+// Note that keysBuffer is, in reality, a char** populated by the host
+HOST_IFACE_FUNC
+void __faasm_s3_list_keys(const char* bucketName,
+                          void* keysBuffer,
+                          int* keysBufferLens);
+
+HOST_IFACE_FUNC
+int __faasm_s3_add_key_bytes(const char* bucketName,
+                             const char* keyName,
+                             void* keyBuffer,
+                             int keyBufferLen);
+
+// Note that keyBuffer is, in reality, a uint8_t** populated by the host
+HOST_IFACE_FUNC
+int __faasm_s3_get_key_bytes(const char* bucketName,
+                             const char* keyName,
+                             void* keyBuffer,
+                             int* keyBufferLen);
 #endif

libfaasm/libfaasm.imports

@@ -42,5 +42,13 @@ __faasm_sm_critical_local_end
 # Migration
 __faasm_migrate_point
 
+# S3
+__faasm_s3_get_num_buckets
+__faasm_s3_list_buckets
+__faasm_s3_get_num_keys
+__faasm_s3_list_keys
+__faasm_s3_add_key_bytes
+__faasm_s3_get_key_bytes
+
 # Test
 __faasm_host_interface_test

tasks/func.py

@@ -179,8 +179,16 @@ def local(ctx, clean=False, debug=False):
     """
     Compile all functions used in the tests
     """
+    # Mixing users that use the threads sysroot and the non-threads sysroot
+    # seems to give some rather obscure compilation errors, so just make sure
+    # to separate them
+
+    # Non-threaded users
     user(ctx, "demo", clean, debug)
     user(ctx, "errors", clean, debug)
     user(ctx, "mpi", clean, debug)
+    user(ctx, "s3", clean, debug)
+
+    # Threaded users
     user(ctx, "omp", clean, debug)
     user(ctx, "threads", clean, debug)