diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c8f224..06f6a9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Get the code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: "Set version variables" @@ -34,17 +34,17 @@ jobs: echo "PYTHON_VERSION=$PYTHON_VER" >> $GITHUB_ENV echo "EXAMPLES_VERSION=${CPP_VER}_${PYTHON_VER}" >> $GITHUB_ENV - name: "Set up QEMU" - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: "Set up Docker Buildx" - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: "Log in to DockerHub" - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: faasm.azurecr.io username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }} password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }} - name: "Build and push examples-build image" - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: push: true file: docker/build.dockerfile @@ -54,13 +54,3 @@ jobs: PYTHON_VERSION=${{ env.PYTHON_VERSION }} context: . tags: faasm.azurecr.io/examples-build:${{ env.EXAMPLES_VERSION }} - - name: "Build and push examples-run image" - uses: docker/build-push-action@v3 - with: - push: true - file: docker/run.dockerfile - build-args: | - EXAMPLES_VERSION=${{ env.EXAMPLES_VERSION }} - FAASM_VERSION=${{ env.FAASM_VERSION }} - context: . - tags: faasm.azurecr.io/examples-run:${{ env.FAASM_VERSION }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0136fad..e4e1628 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 # Check if any of the submodules have been modified - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@v3 id: filter with: filters: | @@ -57,7 +57,7 @@ jobs: git config --global --add safe.directory "$GITHUB_WORKSPACE" echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT - - uses: actions/cache/restore@v3.3.1 + - uses: actions/cache/restore@v4 id: wasm-cache with: path: ./wasm @@ -109,7 +109,7 @@ jobs: mv /usr/local/faasm/wasm ./wasm # If we are here we _always_ want to overwrite the cache - name: "Prepare WASM cache" - uses: actions/cache/save@v3.3.1 + uses: actions/cache/save@v4 if: always() with: path: ./wasm @@ -133,7 +133,7 @@ jobs: env: CONAN_CACHE_MOUNT_SOURCE: ~/.conan FAASM_INI_FILE: ./faasm.ini - FAASM_VERSION: 0.20.1 + FAASM_VERSION: 0.21.1 FAASM_WASM_VM: ${{ matrix.faasm_wasm_vm }} steps: - uses: csegarragonz/set-compose-version-action@main @@ -175,7 +175,7 @@ jobs: echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT - name: "Get WASM cache" - uses: actions/cache/restore@v3.3.1 + uses: actions/cache/restore@v4 id: cpp-wasm-cache with: path: ./wasm @@ -184,9 +184,6 @@ jobs: - name: "Upload all WASM files to the cluster" run: ./bin/inv_wrapper.sh wasm.upload - name: "Run MPI kernels" - # TODO: MPI kernels still not run in WAMR because there's a seg. fault - # with the `exit` syscall - if: "contains(env.FAASM_WASM_VM, 'wavm')" timeout-minutes: 2 run: | # The global MPI kernel stopped working when enabling SIMD diff --git a/FAASM_VERSION b/FAASM_VERSION deleted file mode 100644 index c5523bd..0000000 --- a/FAASM_VERSION +++ /dev/null @@ -1 +0,0 @@ -0.17.0 diff --git a/docker-compose.yml b/docker-compose.yml index 1d72037..57ad220 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,47 +1,6 @@ version: "3" services: - redis: - image: faasm.azurecr.io/redis:${EXAMPLES_RUN_VERSION} - - planner: - image: faasm.azurecr.io/planner:0.11.0 - environment: - - LOG_LEVEL=info - - PLANNER_PORT=8081 - - minio: - image: faasm.azurecr.io/minio:${EXAMPLES_RUN_VERSION} - environment: - MINIO_ROOT_USER: minio - MINIO_ROOT_PASSWORD: minio123 - volumes: - - ./dev/minio/data:/data/minio - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 5s - timeout: 3s - retries: 3 - - run: - depends_on: - - redis - - minio - image: faasm.azurecr.io/examples-run:${EXAMPLES_RUN_VERSION} - working_dir: /usr/local/code/faasm - stdin_open: true - tty: true - volumes: - - ./:/code/examples - - ./dev/faasm-local/:/usr/local/faasm - environment: - - DEFAULT_MPI_WORLD_SIZE=4 - - LOG_LEVEL=info - - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib - - REDIS_QUEUE_HOST=redis - - REDIS_STATE_HOST=redis - - FAASM_WASM_VM=${FAASM_WASM_VM:-wavm} - build: image: faasm.azurecr.io/examples-build:${EXAMPLES_BUILD_VERSION} working_dir: /code/examples diff --git a/docker/run.dockerfile b/docker/run.dockerfile deleted file mode 100644 index 1b12625..0000000 --- a/docker/run.dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG EXAMPLES_VERSION -ARG FAASM_VERSION -FROM faasm.azurecr.io/examples-build:${EXAMPLES_VERSION} as build - -# Prepare shared data for the tests -RUN cd /code/examples \ - && source venv/bin/activate \ - && inv data - -FROM faasm.azurecr.io/cli:${FAASM_VERSION} - -COPY --from=build /code/examples /code/examples -COPY --from=build /usr/local/faasm/wasm /usr/local/faasm/wasm diff --git a/requirements.txt b/requirements.txt index 33fa976..2dd9b90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ black>=22.3.0 -faasmctl>=0.23.0 +faasmctl>=0.28.0 flake8>=4.0.1 invoke>=1.7.1 +PyYAML>=6.0.1 diff --git a/tasks/cli.py b/tasks/cli.py index bd8a433..5e07bf8 100644 --- a/tasks/cli.py +++ b/tasks/cli.py @@ -8,7 +8,6 @@ DEV_FAASM_LOCAL, EXAMPLES_BUILD_IMAGE_NAME, PROJ_ROOT, - get_faasm_version, get_version, ) @@ -40,12 +39,7 @@ def cli(ctx, service, clean=False): run("docker rm -f {}".format(tmp_ctr_name), shell=True, check=True) build_env = environ.copy() - build_env.update( - { - "EXAMPLES_RUN_VERSION": get_faasm_version(), - "EXAMPLES_BUILD_VERSION": get_version(), - } - ) + build_env.update({"EXAMPLES_BUILD_VERSION": get_version()}) docker_cmd = "docker compose up -d --no-recreate {}".format(service) run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT, env=build_env) @@ -56,12 +50,7 @@ def cli(ctx, service, clean=False): @task() def stop(ctx): build_env = environ.copy() - build_env.update( - { - "EXAMPLES_RUN_VERSION": get_faasm_version(), - "EXAMPLES_BUILD_VERSION": get_version(), - } - ) + build_env.update({"EXAMPLES_BUILD_VERSION": get_version()}) run( "docker compose down", shell=True, diff --git a/tasks/env.py b/tasks/env.py index 272913a..6a265cf 100644 --- a/tasks/env.py +++ b/tasks/env.py @@ -3,8 +3,7 @@ from faasmtools.env import get_version as get_cpp_version from os import environ from os.path import dirname, abspath, join -from re import search as re_search -from subprocess import run +from yaml import safe_load PROJ_ROOT = dirname(dirname(abspath(__file__))) EXAMPLES_IN_DOCKER_ROOT = "/code/examples" @@ -69,55 +68,22 @@ def get_python_version(): def get_faasm_version(): """ - Get the version of the python submodule + Get the version of the Faasm dependency """ - ver_file = join(PROJ_ROOT, "FAASM_VERSION") - - with open(ver_file, "r") as fh: - version = fh.read() - - version = version.strip() - return version - - -def get_faabric_version(old_faasm_ver, new_faasm_ver): - """ - Get the faabric version by `wget`-ing the FAASM version file for the tagged - Faasm version - """ - - def do_get_ver(faasm_ver): - tmp_file = "/tmp/faabric_version" - wget_cmd = [ - "wget", - "-O {}".format(tmp_file), - "https://raw.githubusercontent.com/faasm/faasm/v{}/.env".format( - faasm_ver - ), - ] - wget_cmd = " ".join(wget_cmd) - out = run(wget_cmd, shell=True, capture_output=True) - assert out.returncode == 0 - - with open(tmp_file, "r") as fh: - ver = re_search(r"planner:([0-9\.]*)", fh.read()).groups(1)[0] - - return ver - - return do_get_ver(old_faasm_ver), do_get_ver(new_faasm_ver) + yaml_path = join(PROJ_ROOT, ".github", "workflows", "tests.yml") + return safe_load(open(yaml_path, "r"))["jobs"]["run-examples-faasmctl"][ + "env" + ]["FAASM_VERSION"] -def get_version(name="build"): +def get_version(): """ Get version for the examples repository The version depends only on the version of the different cross-compilation - toolchains we use, in this case CPP and Python, and Faasm + toolchains we use, in this case CPP and Python """ - if name == "build": - return "{}_{}".format(get_cpp_version(), get_python_version()) - if name == "run": - return "{}".format(get_faasm_version()) + return "{}_{}".format(get_cpp_version(), get_python_version()) def in_docker(): diff --git a/tasks/git.py b/tasks/git.py index 1c9dc59..7924a30 100644 --- a/tasks/git.py +++ b/tasks/git.py @@ -3,16 +3,15 @@ from tasks.env import ( EXAMPLES_BUILD_IMAGE_NAME, PROJ_ROOT, - get_faabric_version, get_faasm_version, get_version, ) VERSIONED_FILES = { - "faasm": ["FAASM_VERSION", ".github/workflows/tests.yml"], - "faabric": ["docker-compose.yml", ".github/workflows/tests.yml"], + "faasm": [".github/workflows/tests.yml"], "cpp": [".github/workflows/tests.yml"], "python": [".github/workflows/tests.yml"], + "faasmctl": ["requirements.txt"], } @@ -41,9 +40,9 @@ def tag(ctx, force=False): @task def bump(ctx, submodule, ver=None): """ - Bump a submodule's dependency: `faasm`, `cpp`, or `python` + Bump a submodule's dependency: `faasm`, `cpp`, `python`, or `faasmctl` """ - allowed_submodules = ["faasm", "cpp", "python"] + allowed_submodules = ["faasm", "cpp", "python", "faasmctl"] if submodule not in allowed_submodules: print("Unrecognised submodule: {}".format(allowed_submodules)) print("Submodule must be one in: {}".format(allowed_submodules)) @@ -61,12 +60,6 @@ def bump(ctx, submodule, ver=None): sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f) run(sed_cmd, shell=True, check=True) - # Also update the planner version accordingly - old_fabric_ver, new_faabric_ver = get_faabric_version(old_ver, new_ver) - for f in VERSIONED_FILES["faabric"]: - sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f) - run(sed_cmd, shell=True, check=True) - else: new_ver = get_version("build") grep_cmd = "grep '{}' .github/workflows/tests.yml".format( @@ -83,3 +76,8 @@ def bump(ctx, submodule, ver=None): for f in VERSIONED_FILES[submodule]: sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f) run(sed_cmd, shell=True, check=True) + + +@task +def foo(ctx): + print(get_faasm_version())