From ff0969f30292be9975c936d69d94faaa7003facf Mon Sep 17 00:00:00 2001
From: Deralden <121487413+Deralden@users.noreply.github.com>
Date: Fri, 21 Jun 2024 15:33:16 +0300
Subject: [PATCH] Github actions implementation (#1821)
* dev
* Delete .circleci directory
* Update master.yml
* Update dev.yml
---
.circleci/config.yml | 71 -------------
.github/workflows/dev.yml | 189 +++++++++++++++++++++++++++++++++++
.github/workflows/master.yml | 189 +++++++++++++++++++++++++++++++++++
README.md | 2 -
4 files changed, 378 insertions(+), 73 deletions(-)
delete mode 100644 .circleci/config.yml
create mode 100644 .github/workflows/dev.yml
create mode 100644 .github/workflows/master.yml
diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index ab4191691..000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,71 +0,0 @@
----
-version: 2
-jobs:
- build:
- docker:
- - image: circleci/python:3.7-node
- - image: circleci/mysql:8.0.21
- command: [--default-authentication-plugin=mysql_native_password]
- environment:
- MYSQL_ROOT_PASSWORD: unittest_pwd
- MYSQL_HOST: '%'
- MYSQL_DATABASE: unittest_db
- MYSQL_USER: unittest_root
- MYSQL_PASSWORD: unittest_pwd
- working_directory: ~/singnet
- steps:
- - checkout
- - run:
- name: Generating checksum for all requirements file
- command: |
- shasum */requirement*.txt > /tmp/checksum_files_list
- sudo chown -R circleci:circleci /usr/local/bin
- sudo chown circleci:circleci -R /usr/local/lib/python3.7
- - run:
- name: Revert the permissions
- command: |
- sudo chown root:root -R /usr/local/bin
- sudo chown root:root -R /usr/local/lib/python3.7
- - run:
- # Our primary container isn't MYSQL so run a sleep command until it's ready.
- name: Waiting for MySQL to be ready
- command: |
- for i in `seq 1 30`;
- do
- nc -z 127.0.0.1 3306 && echo Success && exit 0
- echo -n .
- sleep 1
- done
- echo Failed waiting for MySQL && exit 1
- - run:
- name: Install dependencies
- command: |
- for requirement_file in `ls */*/requirement*.txt`;do sudo pip install -r $requirement_file --prefer-binary ;done
- for requirement_file in `ls */requirement*.txt`;do sudo pip install -r $requirement_file --prefer-binary ;done
- for package_file in `ls */package*.json | cut -d '/' -f1`;do sudo npm install $package_file/;done
- - run:
- name: Install required packages.
- command: |
- sudo apt update --allow-releaseinfo-change
- sudo apt install unzip -y
- wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.6.2.2472-linux.zip
- unzip sonar-scanner-cli-4.6.2.2472-linux.zip
- sudo apt install default-mysql-client
- sudo pip install coverage coveralls pytest alembic
- mysql -h 127.0.0.1 -u root -punittest_pwd -e "create database registry_unittest_db"
- mysql -h 127.0.0.1 -u root -punittest_pwd -e "create database verification_unittest_db"
- mysql -h 127.0.0.1 -u root -punittest_pwd -e "GRANT ALL PRIVILEGES ON *.* TO 'unittest_root'@'%';"
- export PYTHONPATH=$PWD
- for alembic_file in `ls */alembic*.ini | cut -d '/' -f1`;do cd $alembic_file; echo $alembic_file; alembic upgrade head; cd .. ;done
- mysql -h 127.0.0.1 -u unittest_root -punittest_pwd unittest_db < orchestrator/testcases/script.sql
- - run:
- name: Run unit tests and integration test
- command: |
- PYTHONPATH=$PWD python -m coverage run -m pytest */testcases/integration_testcases/*.py */testcases/unit_testcases/*.py */testcases/unit_testcases/*/*.py */testcases/functional_testcases/*.py
- python -m coverage report -m
- coverage xml -i
- sonar-scanner-4.6.2.2472-linux/bin/sonar-scanner -Dsonar.host.url=https://sonarqube.singularitynet.io -Dsonar.login=${SONAR_TOKEN} $SONAR_SCANNER_OPTS_CUSTOM
- - save_cache:
- key: dependency-cache-{{ checksum "/tmp/checksum_files_list" }}
- paths:
- - /usr/local/lib/python3.7/site-packages
\ No newline at end of file
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
new file mode 100644
index 000000000..caf40a666
--- /dev/null
+++ b/.github/workflows/dev.yml
@@ -0,0 +1,189 @@
+name: tests_development
+on:
+ # push:
+ # branches: [ "development" ]
+ pull_request:
+ branches: [ "development" ]
+ workflow_dispatch:
+
+jobs:
+ run_tests_development:
+ runs-on: ubuntu-latest
+ # container: node:20-bookworm
+ services:
+ mysql:
+ image: mysql:8.0.21
+ env:
+ MYSQL_ROOT_PASSWORD: unittest_pwd
+ MYSQL_HOST: '%'
+ MYSQL_DATABASE: unittest_db
+ MYSQL_USER: unittest_root
+ MYSQL_PASSWORD: unittest_pwd
+ ports:
+ - 3306:3306
+
+ steps:
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 18
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.7'
+
+ - name: install pip
+ run: |
+ sudo apt update
+ sudo apt install -y python3-pip netcat-traditional socat iputils-ping
+
+ - name: install packs
+ run: |
+ export PIP_BREAK_SYSTEM_PACKAGES=1
+ sudo apt update --allow-releaseinfo-change
+ sudo apt install unzip -y
+ wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.6.2.2472-linux.zip
+ unzip sonar-scanner-cli-4.6.2.2472-linux.zip
+ sudo apt install default-mysql-client -y
+ # pip3 install coverage coveralls pytest alembic
+ pip3 install pytest setuptools
+
+ - name: clone repo
+ uses: actions/checkout@v3
+
+ - name: check db
+ run: |
+ for i in `seq 1 30`;
+ do
+ nc -z 172.17.0.1 3306 && echo Success && exit 0
+ echo -n .
+ sleep 1
+ done
+ echo Failed waiting for MySQL && exit 1
+
+ - name: eth_event_listener layers packages test
+ run: |
+ pip3 install -r lambda_layers/eth_event_listener/requirements.txt
+
+ - name: install other side packages and layers
+ run: |
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r registry/migration/requirements.txt
+
+ - name: install npm packages
+ run: |
+ # export PIP_BREAK_SYSTEM_PACKAGES=1
+ # export CFLAGS="-I${PREFIX}/include/python3.11/cpython"
+ # for requirement_file in `ls */*/requirement*.txt`;do pip3 install -r $requirement_file ;done
+ # for requirement_file in `ls */requirement*.txt`;do echo "=============$requirement_file=================="; pip3 install -r $requirement_file ;done
+ # pip3 install -r reqs.txt
+ for package_file in `ls */package*.json | cut -d '/' -f1`;do npm install $package_file/;done
+
+ - name: create db
+ run: |
+ # export PIP_BREAK_SYSTEM_PACKAGES=1
+ # socat TCP-LISTEN:3306,fork TCP:mysql:3306 &
+ # mysql -h 172.17.0.1 -u root -punittest_pwd -e "create database registry_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "create database registry_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "create database verification_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "GRANT ALL PRIVILEGES ON *.* TO 'unittest_root'@'%';"
+ export PYTHONPATH=$PWD
+ for alembic_file in `ls */alembic*.ini | cut -d '/' -f1`;do cd $alembic_file; echo $alembic_file; alembic upgrade head; cd .. ;done
+ mysql -h 127.0.0.1 -P 3306 -u unittest_root -punittest_pwd unittest_db < orchestrator/testcases/script.sql
+
+ - name: common tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r common/requirements.txt
+
+ - name: dapp_user tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r dapp_user/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest dapp_user/testcases/unit_testcases/*.py dapp_user/testcases/functional_testcases/*.py
+
+ - name: notification tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r notification/requirements.txt
+
+ - name: payments tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r payments/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest payments/testcases/unit_testcases/*.py
+
+ - name: service_status tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r service_status/requirements.txt
+ pip3 install --upgrade protobuf
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest service_status/testcases/unit_testcases/*.py service_status/testcases/functional_testcases/*.py
+
+ - name: utility tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install setuptools==65.5.0
+ pip3 install wheel==0.38.0
+ pip3 install -r utility/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest utility/testcases/functional_testcases/*.py
+
+ - name: contract_api tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r contract_api/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest contract_api/testcases/unit_testcases/*.py contract_api/testcases/unit_testcases/*/*.py contract_api/testcases/functional_testcases/*.py
+
+ - name: event_pubsub tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r event_pubsub/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest event_pubsub/testcases/unit_testcases/*.py
+
+ - name: orchestrator tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r orchestrator/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest orchestrator/testcases/integration_testcases/*.py orchestrator/testcases/unit_testcases/*.py
+
+ - name: registry tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r registry/migration/requirements.txt
+ pip3 install -r registry/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest registry/testcases/unit_testcases/*.py registry/testcases/functional_testcases/*.py
+
+ - name: signer tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r signer/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest signer/testcases/integration_testcases/*.py
+
+ - name: verification tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r verification/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest verification/testcases/unit_testcases/*.py verification/testcases/functional_testcases/*.py
+
+ - name: wallets tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r wallets/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest wallets/testcases/integration_testcases/*.py wallets/testcases/unit_testcases/*.py
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
new file mode 100644
index 000000000..c77b21cd8
--- /dev/null
+++ b/.github/workflows/master.yml
@@ -0,0 +1,189 @@
+name: tests_master
+on:
+ # push:
+ # branches: [ "master" ]
+ pull_request:
+ branches: [ "master" ]
+ workflow_dispatch:
+
+jobs:
+ run_tests_master:
+ runs-on: ubuntu-latest
+ # container: node:20-bookworm
+ services:
+ mysql:
+ image: mysql:8.0.21
+ env:
+ MYSQL_ROOT_PASSWORD: unittest_pwd
+ MYSQL_HOST: '%'
+ MYSQL_DATABASE: unittest_db
+ MYSQL_USER: unittest_root
+ MYSQL_PASSWORD: unittest_pwd
+ ports:
+ - 3306:3306
+
+ steps:
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 18
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.7'
+
+ - name: install pip
+ run: |
+ sudo apt update
+ sudo apt install -y python3-pip netcat-traditional socat iputils-ping
+
+ - name: install packs
+ run: |
+ export PIP_BREAK_SYSTEM_PACKAGES=1
+ sudo apt update --allow-releaseinfo-change
+ sudo apt install unzip -y
+ wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.6.2.2472-linux.zip
+ unzip sonar-scanner-cli-4.6.2.2472-linux.zip
+ sudo apt install default-mysql-client -y
+ # pip3 install coverage coveralls pytest alembic
+ pip3 install pytest setuptools
+
+ - name: clone repo
+ uses: actions/checkout@v3
+
+ - name: check db
+ run: |
+ for i in `seq 1 30`;
+ do
+ nc -z 172.17.0.1 3306 && echo Success && exit 0
+ echo -n .
+ sleep 1
+ done
+ echo Failed waiting for MySQL && exit 1
+
+ - name: eth_event_listener layers packages test
+ run: |
+ pip3 install -r lambda_layers/eth_event_listener/requirements.txt
+
+ - name: install other side packages and layers
+ run: |
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r registry/migration/requirements.txt
+
+ - name: install npm packages
+ run: |
+ # export PIP_BREAK_SYSTEM_PACKAGES=1
+ # export CFLAGS="-I${PREFIX}/include/python3.11/cpython"
+ # for requirement_file in `ls */*/requirement*.txt`;do pip3 install -r $requirement_file ;done
+ # for requirement_file in `ls */requirement*.txt`;do echo "=============$requirement_file=================="; pip3 install -r $requirement_file ;done
+ # pip3 install -r reqs.txt
+ for package_file in `ls */package*.json | cut -d '/' -f1`;do npm install $package_file/;done
+
+ - name: create db
+ run: |
+ # export PIP_BREAK_SYSTEM_PACKAGES=1
+ # socat TCP-LISTEN:3306,fork TCP:mysql:3306 &
+ # mysql -h 172.17.0.1 -u root -punittest_pwd -e "create database registry_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "create database registry_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "create database verification_unittest_db"
+ mysql -h 127.0.0.1 -P 3306 -u root -punittest_pwd -e "GRANT ALL PRIVILEGES ON *.* TO 'unittest_root'@'%';"
+ export PYTHONPATH=$PWD
+ for alembic_file in `ls */alembic*.ini | cut -d '/' -f1`;do cd $alembic_file; echo $alembic_file; alembic upgrade head; cd .. ;done
+ mysql -h 127.0.0.1 -P 3306 -u unittest_root -punittest_pwd unittest_db < orchestrator/testcases/script.sql
+
+ - name: common tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r common/requirements.txt
+
+ - name: dapp_user tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r dapp_user/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest dapp_user/testcases/unit_testcases/*.py dapp_user/testcases/functional_testcases/*.py
+
+ - name: notification tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r notification/requirements.txt
+
+ - name: payments tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r payments/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest payments/testcases/unit_testcases/*.py
+
+ - name: service_status tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r service_status/requirements.txt
+ pip3 install --upgrade protobuf
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest service_status/testcases/unit_testcases/*.py service_status/testcases/functional_testcases/*.py
+
+ - name: utility tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install setuptools==65.5.0
+ pip3 install wheel==0.38.0
+ pip3 install -r utility/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest utility/testcases/functional_testcases/*.py
+
+ - name: contract_api tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r contract_api/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest contract_api/testcases/unit_testcases/*.py contract_api/testcases/unit_testcases/*/*.py contract_api/testcases/functional_testcases/*.py
+
+ - name: event_pubsub tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r event_pubsub/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest event_pubsub/testcases/unit_testcases/*.py
+
+ - name: orchestrator tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r orchestrator/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest orchestrator/testcases/integration_testcases/*.py orchestrator/testcases/unit_testcases/*.py
+
+ - name: registry tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r lambda_layers/web3_and_grpc_libraries/requirements.txt
+ pip3 install -r lambda_layers/common_python_libraries/requirements.txt
+ pip3 install -r registry/migration/requirements.txt
+ pip3 install -r registry/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest registry/testcases/unit_testcases/*.py registry/testcases/functional_testcases/*.py
+
+ - name: signer tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r signer/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest signer/testcases/integration_testcases/*.py
+
+ - name: verification tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r verification/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest verification/testcases/unit_testcases/*.py verification/testcases/functional_testcases/*.py
+
+ - name: wallets tests
+ run: |
+ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+ export AWS_XRAY_SDK_ENABLED=false
+ pip3 install -r wallets/requirements.txt
+ PYTHONPATH=$PWD python3 -m coverage run -m pytest wallets/testcases/integration_testcases/*.py wallets/testcases/unit_testcases/*.py
diff --git a/README.md b/README.md
index 418e6b717..aba6c62ae 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,5 @@
# snet-marketplace-service
-[](https://circleci.com/gh/singnet/snet-marketplace-service)
-[](https://coveralls.io/github/singnet/snet-marketplace-service?branch=master)
## Getting Started
### SingularityNET Marketplace Services
The best way to get started is by understanding the marketplace services that powers the decentralized Blockchain marketplace. Here is a high level view on how SingularityNet's services works.