Add more commands to the encointer-client docker image and test them …

…in CI (#385)

* [CI] put all scripts into the docker container

* [CI] fix bootstrap commant to run int docker

* [CI] fix phase.py to work in docker

* [CI] fix faucet.py to run in docker

* [CI] add checking the botstats to the docker

* try making the bot-community.py script work, but it fails to connect to the faucet service

* add some documentation

* fix connecting to the faucet

* fix faucet

* remove comment that it doesn't work

* docker build now uses the current dir to look for the encointer-client again

* add integration tests for the docker

* remove -ti flag for docker runs

* [CI] remove unnecessary steps

* [CI] fix matrix

* [CI] downgrade to ubuntu 22.04

* [CI] fix syntax

* [CI] update rust cache actions

* remove unnecessary python setup

* don't fail fast in docker integration tests

* try fixing the node command

* download past artifacts for debugging

* disable previous steps

* fix CI run for artifact download

* fix docker build

* fix matrix tests

* try to avoid port collision

* unify interface of scripts regarding url

* fix bootstrap script and disable business docker CI for now

* fix entry script

* fix docker cache

* fix new script interface

* fix new script interface 2

* fix documentation and remove unnecessary port exposure

* document in which cases url accepts gesell as value

* enable the rest of the CI again

.github/workflows/ci.yml

@@ -21,13 +21,13 @@ env:
 jobs:
   cancel_previous_runs:
     name: Cancel Previous Runs
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: styfle/[email protected]
         with:
           access_token: ${{ secrets.GITHUB_TOKEN }}
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
 
@@ -39,15 +39,9 @@ jobs:
       - name: Setup Rust toolchain
         run: rustup show
 
-      - name: Cache Rust Dependecies
-        uses: actions/cache@v3
+      - uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-            enclave/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          shared-key: "release"
 
       - name: Build
         run: cargo build --release
@@ -63,7 +57,7 @@ jobs:
           path: target/release/encointer-client-notee
 
   build-try-runtime-and-benchmarks:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
 
@@ -75,15 +69,9 @@ jobs:
       - name: Setup Rust toolchain
         run: rustup show
 
-      - name: Cache Rust Dependecies
-        uses: actions/cache@v3
+      - uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-            enclave/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          shared-key: "release"
 
       - name: Build
         run: cargo build --release --features try-runtime,runtime-benchmarks
@@ -100,7 +88,7 @@ jobs:
 
   build-runtimes:
     name: Build Runtimes
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         runtime: [ "encointer-node-notee" ]
@@ -115,6 +103,7 @@ jobs:
           restore-keys: |
             srtool-target-${{ matrix.runtime }}-
             srtool-target-
+
       - name: Srtool build
         id: srtool_build
         uses: chevdor/[email protected]
@@ -166,7 +155,6 @@ jobs:
             ${{ matrix.runtime }}-metadata.json
       #            ${{ matrix.runtime }}-diff.txt
 
-
       - name: Upload ${{ matrix.runtime }} runtime
         uses: actions/upload-artifact@v4
         with:
@@ -176,7 +164,7 @@ jobs:
             ${{ steps.srtool_build.outputs.wasm_compressed }}
 
   unit-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
 
@@ -188,15 +176,9 @@ jobs:
       - name: Setup Rust toolchain
         run: rustup show
 
-      - name: Cache Rust Dependecies
-        uses: actions/cache@v3
+      - uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-            enclave/target
-          key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+          shared-key: "debug"
 
       - name: cargo test
         run: cargo test --all
@@ -206,7 +188,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
+        os: [ ubuntu-22.04 ]
         rust: [ stable ]
         rust-target: [ x86_64-unknown-linux-gnu ]
         check: [ +nightly fmt --all -- --check, clippy -p encointer-client-notee ]
@@ -235,13 +217,13 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          key: ${{ matrix.rust-target }}-${{ matrix.check }}
+          shared-key: "debug"
 
       - name: ${{ matrix.check }}
         run: cargo ${{ matrix.check }}
 
   cargo-toml-fmt:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     container: "tamasfe/taplo:0.7.0-alpine"
     steps:
       - uses: actions/checkout@v3
@@ -255,7 +237,7 @@ jobs:
 
   integration-test:
     name: ${{ matrix.test }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: build
     strategy:
       matrix:
@@ -310,10 +292,120 @@ jobs:
         working-directory: ./scripts/ci
         run: source ./init_env.sh && ./${{ matrix.test }}
 
+  integration-test-docker:
+    name: ${{ matrix.test }}
+    runs-on: ubuntu-22.04
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - bootstrap_demo_community.py --signer //Bob --test
+          - bot-community-test -f http://host.docker.internal:5000/api
+          # Todo: #386
+          # - test-register-businesses -f http://host.docker.internal:5000/api
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: encointer-node-notee-${{ github.sha }}
+          # for debugging the integration tests, we can just download an image from a previous run
+      #          name: encointer-node-notee-4d0313f614223edf63cd0a5f0ad3d5ce16b81e6c
+      #          github-token: ${{ github.token }}
+      #          run-id: 12787265993
+
+      - name: download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: encointer-client-notee-${{ github.sha }}
+          # for debugging the integration tests, we can just download an image from a previous run
+      #          name: encointer-client-notee-4d0313f614223edf63cd0a5f0ad3d5ce16b81e6c
+      #          github-token: ${{ github.token }}
+      #          run-id: 12787265993
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache-node
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Docker build client
+        run: |
+          docker build \
+            --cache-from=type=gha \
+            -t encointer-client-test .
+
+      - name: Docker build node
+        run: |
+          docker build \
+          --cache-from=type=gha \
+          -f Dockerfile-node \
+          -t encointer-node-test . 
+
+      - name: Set up ipfs
+        uses: ibnesayeed/setup-ipfs@master
+        with:
+          run_daemon: true
+
+      - name: start dev node
+        run: |
+          docker run -p 30333:30333 -p 9944:9944 -p 9615:9615 \
+            encointer-node-test \
+            --dev \
+            --enable-offchain-indexing=true \
+            --rpc-methods=unsafe \
+            -lencointer=debug,parity_ws=warn \
+            --rpc-external &
+
+      - name: start faucet service
+        run: |
+          docker run -p 5000:5000 \
+            --add-host host.docker.internal:host-gateway \
+            encointer-client-test faucet.py \
+            -u ws://host.docker.internal \
+            --port 9944 &
+
+      - name: start phase accelerator service
+        run: |
+          docker run \
+            --add-host host.docker.internal:host-gateway \
+            encointer-client-test phase.py \
+            -u ws://host.docker.internal \
+            --port 9944 --idle-blocks 3 &
+
+      - name: Test ${{ matrix.test }}
+        run: |
+          docker run \
+            --add-host host.docker.internal:host-gateway \
+            encointer-client-test ${{ matrix.test }} \
+            -u ws://host.docker.internal \
+            --port 9944 \
+
   release:
     name: Draft Release
     if: startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [ build, unit-tests, check, integration-test ]
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}
@@ -357,7 +449,7 @@ jobs:
 
   publish-runtimes:
     name: Publish Runtimes
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [ release, build-runtimes ]
     strategy:
       matrix:

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 LABEL description="This is the 2nd stage: a very small image where we copy the Substrate binary."
 
-RUN apt-get update && \ 
+RUN apt-get update && \
 apt-get install -y jq python3 python3-pip
 
 RUN python3 -m pip install --upgrade pip
@@ -24,9 +24,17 @@ COPY encointer-client-notee /
 
 RUN mkdir /client
 COPY client/py_client /py_client
+COPY client/test-data /test-data
+
+# all python scripts (some of them aren supported by the entryfile.sh yet).
 COPY client/bootstrap_demo_community.py /
+COPY client/bot-community.py /
+COPY client/bot-stats-golden.csv /
 COPY client/cli.py /
-COPY client/test-data /test-data
+COPY client/faucet.py /
+COPY client/phase.py /
+COPY client/typedefs.json /
+COPY client/register-random-businesses-and-offerings.py /
 
 RUN chmod +x /encointer-client-notee
 #RUN chmod +x /usr/local/bin/healthcheck9933.sh
@@ -40,7 +48,7 @@ RUN ldd /encointer-client-notee && \
 #	rm -rf /usr/bin /usr/sbin /usr/share/man
 
 #USER encointer
-EXPOSE 30333 9933 9944 9615
+EXPOSE 30333 9933 9944 9615 5000
 VOLUME ["/data"]
 
 ENTRYPOINT ["/entryscript.sh"]

client/bazaar.py

@@ -30,25 +30,25 @@
 import os
 
 
-
-
 @click.group()
-@click.option('--cid', required=True, help='the community identifier of the community you want to register your business in (11 digits).')
+@click.option('--cid', required=True,
+              help='the community identifier of the community you want to register your business in (11 digits).')
 @click.option('--bizaccount', required=False, help='the account of the owner in ss58 format or raw_seed.')
 @click.option('--price', default='0', help='price of your offering.')
-@click.option('--client', default='../target/release/encointer-client-notee', help='Client binary to communicate with the chain.')
+@click.option('--client', default='../target/release/encointer-client-notee',
+              help='Client binary to communicate with the chain.')
 @click.option('--port', default='9944', help='ws-port of the chain.')
-@click.option('-r', '--remote_chain', default=None, help='choose remote chain: gesell.')
+@click.option('-u', '--url', default='ws://127.0.0.1', help='URL of the chain, or `gesell` alternatively.')
 @click.pass_context
-def cli(ctx, client, port, cid, bizaccount, price, remote_chain):
+def cli(ctx, client, port, cid, bizaccount, price, url):
     ctx.ensure_object(dict)
-    cl = set_local_or_remote_chain(client, port, remote_chain)
+    cl = set_local_or_remote_chain(client, port, url)
     ctx.obj['client'] = cl
     ctx.obj['port'] = port
     ctx.obj['cid'] = cid
     ctx.obj['bizaccount'] = bizaccount
     # ctx.obj['ipfs_local'] = ipfs_local
-    ctx.obj['remote_chain'] = remote_chain
+    ctx.obj['url'] = url
     ctx.obj['price'] = price
 
 
@@ -132,6 +132,7 @@ def register_offering(ctx, specfile):
     except:
         print("error creating an offering entry")
 
+
 @cli.command()
 @click.pass_obj
 def list_businesses(ctx):
@@ -143,6 +144,7 @@ def list_businesses(ctx):
     client = ctx['client']
     print(client.list_businesses(ctx['cid']))
 
+
 @cli.command()
 @click.pass_obj
 def list_offerings(ctx):
@@ -159,5 +161,6 @@ def list_offerings(ctx):
     else:
         print(client.list_offerings_for_business(ctx['cid'], ctx["bizaccount"]))
 
+
 if __name__ == '__main__':
     cli(obj={})

client/bootstrap_demo_community.py

@@ -405,7 +405,7 @@ def test_democracy(client, cid):
 @click.option('--client', default='../target/release/encointer-client-notee',
               help='Client binary to communicate with the chain.')
 @click.option('--signer', help='optional account keypair creating the community')
-@click.option('-u', '--url', default='ws://127.0.0.1', help='URL of the chain.')
+@click.option('-u', '--url', default='ws://127.0.0.1', help='URL of the chain, or `gesell` alternatively.')
 @click.option('-p', '--port', default='9944', help='ws-port of the chain.')
 @click.option('-l', '--ipfs-local', is_flag=True, help='if set, local ipfs node is used.')
 @click.option('-s', '--spec-file', default=f'{TEST_DATA_DIR}{TEST_LOCATIONS_MEDITERRANEAN}',