Skip to content

Commit

Permalink
update CI and code adjustments
Browse files Browse the repository at this point in the history
  • Loading branch information
devProdigy committed Dec 9, 2020
1 parent 6807906 commit 161916c
Show file tree
Hide file tree
Showing 42 changed files with 421 additions and 293 deletions.
84 changes: 58 additions & 26 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ on:
pull_request:

jobs:
test:
code_checks:
runs-on: ubuntu-latest
strategy:
matrix:
os: [ubuntu-latest]
component: [formatting, builder, cli, client, machine, reporters, serializer, server, util, workflow, allelse]
code-check: [black, flakehell]
python-version: [3.7]
steps:
- uses: actions/checkout@v1
Expand All @@ -23,48 +23,72 @@ jobs:
- uses: actions/cache@v2
if: startsWith(runner.os, 'Linux')
with:
path: ~/.cache/pip
key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements/*requirements.txt') }}
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/*requirements.txt') }}
restore-keys: |
${{ runner.os }}-${{ matrix.python-version }}-pip-
- name: Extract branch name (works for both 'pull' and 'push' events)
- name: Extract branch name # works for both 'push' and 'pull_request' events
shell: bash
run: echo "##[set-output name=branch;]$( if [ ${{ github.event_name }} = push ]; then echo ${GITHUB_REF#refs/heads/}; else echo ${{ github.event.pull_request.head.ref }}; fi )"
id: extract_branch

- name: Install
- name: Install test requirements
run: |
pip install --upgrade pip
pip install --upgrade pip-tools
pip uninstall -y aws-sam-cli pyyaml # Workaround for fixing an issue with incompatible package versions
pip install -r requirements/full_requirements.txt
pip install .[tests]
# - name: Get git diff files for Black
# shell: bash
# run: |
# git diff origin/master..origin/"${{ steps.extract_branch.outputs.branch }}" --name-only
# FILE_DIFFS=$(git diff origin/master..origin/${{ steps.extract_branch.outputs.branch }} --name-only | sed -n "/.a/ p")
# echo "FILE_DIFFS=$FILE_DIFFS" >> $GITHUB_ENV
# echo $GITHUB_ENV
# echo 5
# id: python_git_diff_files
pip install -r requirements/test_requirements.txt
- name: Run code quality checks
# if: ${{ env.FILE_DIFFS }}
run: |
FILE_DIFFS=$(git diff origin/master..origin/${{ steps.extract_branch.outputs.branch }} --name-only | sed -n "/.py/ p")
echo "${FILE_DIFFS}"
FILES_TO_CHECK=$(git diff origin/master..origin/${{ steps.extract_branch.outputs.branch }} --name-only --diff-filter=ACM '*py')
echo "> Changed Python files: ${FILES_TO_CHECK}"
if [ -z "${FILE_DIFFS}" ]
if [ -z "${FILES_TO_CHECK}" ]
then
echo "No Python files to check."
echo "> No Python files to check, exiting."
exit 0
fi
if [ ${{ matrix.code-check }} = black ]
then
echo "> Starting Black code-formatting check for files: ${FILES_TO_CHECK}"
black --diff ${FILES_TO_CHECK}
else
git diff origin/master..origin/${{ steps.extract_branch.outputs.branch }} | flakehell lint --diff
black --check "${FILE_DIFFS}"
echo "> Starting Flakehell code-quality check for diff in files: ${FILES_TO_CHECK}"
git diff origin/master..origin/${{ steps.extract_branch.outputs.branch }} -U0 --diff-filter=ACM '*.py' | flakehell lint --diff
fi
test:
runs-on: ubuntu-latest
needs: code_checks
strategy:
matrix:
os: [ubuntu-latest]
component: [builder, cli, client, machine, reporters, serializer, server, util, workflow, allelse]
python-version: [3.7]
steps:
- uses: actions/checkout@v1

- uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
architecture: 'x64'

- uses: actions/cache@v2
if: startsWith(runner.os, 'Linux')
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('**/*requirements.txt') }}
restore-keys: |
${{ env.pythonLocation }}-${{ runner.os }}-${{ matrix.python-version }}-env-
- name: Install
run: |
pip install --upgrade pip==20.1.1
pip install -r requirements/full_requirements.txt
pip install -r requirements/test_requirements.txt
- name: Test ${{ matrix.component }}
run: python setup.py test${{ matrix.component }}

Expand Down Expand Up @@ -100,6 +124,7 @@ jobs:

build-docs:
runs-on: ubuntu-latest
needs: code_checks
steps:
- uses: actions/checkout@v1

Expand All @@ -108,6 +133,13 @@ jobs:
python-version: '3.7' # Version range or exact version of a Python version to use, using semvers version range syntax.
architecture: 'x64'

- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-${{ matrix.python-version }}-pip-new-${{ hashFiles('**/*requirements.txt') }}
restore-keys: |
${{ runner.os }}-${{ matrix.python-version }}-pip-new-
- name: Install deps
run: |
pip install --upgrade pip
Expand Down
68 changes: 5 additions & 63 deletions .github/workflows/master-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,76 +63,18 @@ jobs:
run: |
bash ci/trivy_scan.sh "$IMAGE"
- name: Push base image
run: |
docker push ${{ steps.prep.outputs.base_image }}
- name: Push gordo-client
uses: docker/build-push-action@v2
with:
push: true
context: .
file: ./Dockerfile-Client
build-args: |
BASE_IMAGE=${{ steps.prep.outputs.base_image }}
tags: ${{ steps.prep.outputs.tags_gordo_client }}
labels: |
org.opencontainers.image.title=gordo-client
org.opencontainers.image.description=Gordo client
org.opencontainers.image.url=${{ env.IMAGE_HOME_URL }}
org.opencontainers.image.version=${{ steps.prep.outputs.version }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }}
- name: Push gordo-deploy
uses: docker/build-push-action@v2
with:
push: true
context: .
file: ./Dockerfile-GordoDeploy
build-args: |
BASE_IMAGE=${{ steps.prep.outputs.base_image }}
tags: ${{ steps.prep.outputs.tags_gordo_deploy }}
labels: |
org.opencontainers.image.title=gordo-deploy
org.opencontainers.image.description=Gordo deploy
org.opencontainers.image.url=${{ env.IMAGE_HOME_URL }}
org.opencontainers.image.version=${{ steps.prep.outputs.version }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }}
- name: Push gordo-model-builder
- name: Push gordo-base
uses: docker/build-push-action@v2
with:
push: true
context: .
file: ./Dockerfile-ModelBuilder
build-args: |
BASE_IMAGE=${{ steps.prep.outputs.base_image }}
tags: ${{ steps.prep.outputs.tags_gordo_model_builder }}
labels: |
org.opencontainers.image.title=gordo-model-builder
org.opencontainers.image.description=Gordo model builder
org.opencontainers.image.url=${{ env.IMAGE_HOME_URL }}
org.opencontainers.image.version=${{ steps.prep.outputs.version }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.licenses=${{ env.IMAGE_LICENSE }}
- name: Push gordo-deploy
uses: docker/build-push-action@v2
with:
push: true
context: .
file: ./Dockerfile-ModelServer
file: ./Dockerfile
build-args: |
BASE_IMAGE=${{ steps.prep.outputs.base_image }}
tags: ${{ steps.prep.outputs.tags_gordo_model_server }}
tags: ${{ steps.prep.outputs.tags_gordo_base }}
labels: |
org.opencontainers.image.title=gordo-model-server
org.opencontainers.image.description=Gordo model server
org.opencontainers.image.title=gordo-base
org.opencontainers.image.description=Gordo
org.opencontainers.image.url=${{ env.IMAGE_HOME_URL }}
org.opencontainers.image.version=${{ steps.prep.outputs.version }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,5 @@ venv.bak/
*.pkl
/gordo/_version.py
.python-version
*.DS_Store

36 changes: 36 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,44 @@ RUN pip install -r full_requirements.txt --no-cache-dir
COPY --from=builder /code/dist/gordo-packed.tar.gz .
RUN pip install gordo-packed.tar.gz[full]

# Install GordoDeploy dependencies
ARG HTTPS_PROXY
ARG KUBECTL_VERSION="v1.16.9"
ARG ARGO_VERSION="v2.8.2"

RUN apt-get update && apt-get install -y \
curl \
jq \
&& rm -rf /var/lib/apt/lists/*

#donwload & install kubectl
RUN curl -sSL -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl &&\
chmod +x /usr/local/bin/kubectl

#download & install argo
RUN curl -sSL -o /usr/local/bin/argo https://github.com/argoproj/argo/releases/download/$ARGO_VERSION/argo-linux-amd64 &&\
chmod +x /usr/local/bin/argo

COPY ./run_workflow_and_argo.sh ${HOME}/run_workflow_and_argo.sh

# Baking in example configs for running tests, as docker.client.containers.run
# bind doesn't seem to work correctly for non-root users
# volumes={repo_dir: {"bind": "/home/gordo", "mode": "ro"}},
COPY ./examples ${HOME}/examples
COPY ./resources ${HOME}/resources

# Install ModelBuilder dependencies
ADD build.sh ${HOME}/build.sh

# build.sh (build the model) as executable default command
RUN cp ${HOME}/build.sh /usr/bin/build \
&& chmod a+x /usr/bin/build


# Make gordo own all in its home
RUN chown -R gordo:gordo ${HOME}

# Run things from gordo's home to have write access when needed (e.g. Catboost tmp files)
WORKDIR ${HOME}
# Switch user
USER gordo
7 changes: 0 additions & 7 deletions Dockerfile-Client

This file was deleted.

35 changes: 0 additions & 35 deletions Dockerfile-GordoDeploy

This file was deleted.

13 changes: 0 additions & 13 deletions Dockerfile-ModelBuilder

This file was deleted.

7 changes: 0 additions & 7 deletions Dockerfile-ModelServer

This file was deleted.

29 changes: 24 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ MODEL_SERVER_IMG_NAME := gordo-model-server
CLIENT_IMG_NAME := gordo-client
WORKFLOW_GENERATOR_IMG_NAME := gordo-deploy

.SILENT: code-quality-locally black-check flakehell-check

base:
docker build . -f Dockerfile -t $(BASE_IMG_NAME)

Expand Down Expand Up @@ -95,13 +97,30 @@ sdist:

images: model-builder model-server client

code-quality: flakehell black ## Run code quality tools
code-quality-locally:
@echo "** Make sure that your branch is up to date with origin/master (not to have extra files to check)"
make black-check; make flakehell-check ## run code quality check on changed files in the current branch

black-check: ## run black code formatting check on changed files in the current branch
@$(eval FILES_TO_CHECK=$(shell git diff origin/master --name-only --diff-filter=ACM '*.py'))

flakehell: ## Run flakehell with plugins - only on changed code
git diff origin/master.. | flakehell lint --diff
if [ -z "${FILES_TO_CHECK}" ]; then \
echo "> No Python files to check."; \
else \
echo "> Starting Black code-formatting check for files: ${FILES_TO_CHECK}"; \
black --diff ${FILES_TO_CHECK}; \
fi

black: ## Run black auto code formatter - only on changed code
git diff origin/master.. --name-only | grep '.py' | xargs black --check
flakehell-check: ## run flake8 and its plugins code checks on changes in the current branch
@$(eval FILES_TO_CHECK=$(shell git diff origin/master --name-only --diff-filter=ACM '*.py'))

if [ -z "${FILES_TO_CHECK}" ]; then \
echo "> No Python files to check."; \
else \
echo "> Starting Flakehell code-quality check for diff in files: ${FILES_TO_CHECK}"; \
git diff origin/master -U0 --diff-filter=ACM '*.py' | flakehell lint --diff; \
echo "> 'Flakehell' finished."; \
fi

test:
python setup.py test
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ This section will explain how to start development of Gordo.
```shell script
# create and activate virtualenv. Note: you should use python3.7 (project's tensorflow version is not compatible with python3.8)
# then:
make make install_app_requirements
make install_app_requirements
```

### How to run tests locally
Expand Down
Loading

0 comments on commit 161916c

Please sign in to comment.