Merge pull request #150 from reportportal/rc/5.10.0

Release 5.10.0
reportportal · Oct 2, 2023 · 5cee614 · 5cee614
2 parents 3f8e6a7 + c0e0856
commit 5cee614
Show file tree

Hide file tree

Showing 286 changed files with 8,663 additions and 6,613 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,4 +1,3 @@
 [flake8]
-ignore = E241
 exclude = .git,venv,env,fixtures
-max-line-length = 110
+max-line-length = 119
diff --git a/.github/workflows/dockerhub-release.yaml b/.github/workflows/dockerhub-release.yaml
@@ -0,0 +1,71 @@
+name: Retag RC Docker image
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+env:
+  AWS_REGION: ${{ vars.AWS_REGION }}               # set this to your preferred AWS region, e.g. us-west-1
+  ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}       # set this to your Amazon ECR repository name
+  TARGET_REGISTRY: ${{ vars.TARGET_REGISTRY }}     # set to target regestry (DockerHub, GitHub & etc)
+  TARGET_REPOSITORY: ${{ vars.TARGET_REPOSITORY }} # set to target repository
+  PLATFORMS: ${{ vars.BUILD_PLATFORMS }}           # set target build platforms. By default linux/amd64
+  RELEASE_MODE: ${{ vars.RELEASE_MODE }}
+
+jobs:
+  retag-image:
+    name: Retag and push image 
+    runs-on: ubuntu-latest
+    environment: rc
+    if: github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: 'true'
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.REGESTRY_USERNAME }}
+          password: ${{ secrets.REGESTRY_PASSWORD }}
+
+      - name: Create variables
+        id: vars
+        run: |
+          echo "tag=$(echo '${{ github.event.pull_request.title }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT
+      
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Retag and Push Docker Image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          IMAGE_TAG: ${{ steps.vars.outputs.tag }}
+        run: |
+          docker buildx imagetools create $ECR_REGISTRY/$ECR_REPOSITORY:latest --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG --tag $TARGET_REGISTRY/$TARGET_REPOSITORY:latest
+      
+      - name: Summarize
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          IMAGE_TAG: ${{ steps.vars.outputs.tag }}
+        run: |
+          echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- :whale: Docker image: $TARGET_REGISTRY/$TARGET_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY
+          echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/rc.yaml b/.github/workflows/rc.yaml
@@ -0,0 +1,76 @@
+name: Build RC Docker image
+
+on:
+  push:
+    branches:
+      - "rc/*"
+      - "hotfix/*"
+
+env:
+  AWS_REGION: ${{ vars.AWS_REGION }}         # set this to your preferred AWS region, e.g. us-west-1
+  ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }} # set this to your Amazon ECR repository name
+  PLATFORMS: ${{ vars.BUILD_PLATFORMS }}      # set target build platforms. By default linux/amd64
+  RELEASE_MODE: ${{ vars.RELEASE_MODE }}
+
+jobs:
+  build-and-export:
+    name: Build and export to AWS ECR
+    runs-on: ubuntu-latest
+    environment: rc
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          # role-to-assume: arn:aws:iam::123456789012:role/my-github-actions-role
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: 'true'
+
+      - name: Create variables
+        id: vars
+        run: |
+          echo "version=$(echo '${{ github.ref_name }}' | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p')" >> $GITHUB_OUTPUT
+          echo "tag=$(echo ${{ github.ref_name }}-${{ github.run_number }} | tr '/' '-')" >> $GITHUB_OUTPUT
+          
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build
+        uses: docker/build-push-action@v4
+        env:
+          VERSION: ${{ steps.vars.outputs.version }}
+          IMAGE_TAG: ${{ steps.vars.outputs.tag }}
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        with:
+          context: .
+          push: true
+          build-args: |
+            APP_VERSION=${{ env.VERSION }}
+            RELEASE_MODE=${{ env.RELEASE_MODE }}
+            GITHUB_TOKEN=${{ secrets.GH_TOKEN }}
+          platforms: ${{ env.PLATFORMS }}
+          tags: |
+            ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+            ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest
+      
+      - name: Summarize
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          IMAGE_TAG: ${{ steps.vars.outputs.tag }}
+        run: |
+          echo "## General information about the build:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- :gift: Docker image in Amazon ECR: ecr/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_STEP_SUMMARY
+          echo "- :octocat: The commit SHA from which the build was performed: [$GITHUB_SHA](https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA)" >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -30,9 +30,12 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r ./requirements.txt
-          pip install -r ./requirements_dev.txt
-          python nltk_download.py
+          python -m pip install -r ./requirements.txt
+          python -m pip install -r ./requirements-dev.txt
+          python -m nltk.downloader stopwords
+
+      - name: Checkstyle
+        run: python -m flake8
 
       - name: Test
         run: pytest --cov-config=.coveragerc --cov --cov-report=xml test/ -s -vv

diff --git a/.gitignore b/.gitignore
@@ -7,8 +7,8 @@ __pycache__/
 *.so
 
 # Distribution / packaging
-.idea
 .Python
+.idea
 build/
 develop-eggs/
 dist/
@@ -21,6 +21,8 @@ parts/
 sdist/
 var/
 wheels/
+pip-wheel-metadata/
+share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -39,12 +41,14 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
+.nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
+*.py,cover
 .hypothesis/
 .pytest_cache/
 
@@ -56,6 +60,7 @@ coverage.xml
 *.log
 local_settings.py
 db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -73,11 +78,26 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# IPython
+profile_default/
+ipython_config.py
+
 # pyenv
 .python-version
 
-# celery beat schedule file
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
 celerybeat-schedule
+celerybeat.pid
 
 # SageMath parsed files
 *.sage.py
@@ -103,5 +123,8 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+.dmypy.json
+dmypy.json
 
-docker-compose.yaml
+# Pyre type checker
+.pyre/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,9 @@
+# Changelog
+
+## [Unreleased]
+### Changed
+- Global structure update
+- Deprecated `unique_id` field changed to `test_case_hash`, by @HardNorth
+- `ES_BOOST_UNIQUE_ID` environment variable renamed to `ES_BOOST_TEST_CASE_HASH`, by @HardNorth
+
+## [5.7.2]
diff --git a/Dockerfile b/Dockerfile
@@ -1,50 +1,57 @@
-FROM python:3.10.12
-
-RUN apt-get update && apt-get install -y build-essential && \
-    rm -rf /var/lib/apt/lists/*
-RUN mkdir /backend/
-WORKDIR /backend/
-
-ADD requirements.txt requirements.txt
-
-RUN python -m venv /venv \
-    && /venv/bin/pip install -U pip \
-    && LIBRARY_PATH=/lib:/usr/lib /bin/sh -c "/venv/bin/pip install --no-cache-dir -r requirements.txt"
-
-RUN touch /venv/bin/activate
-RUN /venv/bin/python3 -m nltk.downloader -d /usr/share/nltk_data stopwords
-
-ARG version
-ARG prod
-ARG githubtoken
-
+FROM --platform=${BUILDPLATFORM} python:3.10.12 as test
+RUN apt-get update && apt-get install -y build-essential \
+    && rm -rf /var/lib/apt/lists/* \
+    && python -m venv /venv \
+    && mkdir /build
+ENV VIRTUAL_ENV=/venv
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+WORKDIR /build
 COPY ./ ./
-
+RUN "${VIRTUAL_ENV}/bin/pip" install --upgrade pip \
+    && LIBRARY_PATH=/lib:/usr/lib /bin/sh -c "${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements.txt" \
+    && "${VIRTUAL_ENV}/bin/python3" -m nltk.downloader -d /usr/share/nltk_data stopwords
+RUN "${VIRTUAL_ENV}/bin/pip" install --no-cache-dir -r requirements-dev.txt
 RUN make test-all
-RUN if [ "$prod" = "true" ]; then make release v=$version githubtoken=$githubtoken; else if [ "$version" != "" ]; then make build-release v=$version; fi ; fi
 
-# Multistage
-FROM python:3.10.12-slim
+
+FROM --platform=${BUILDPLATFORM} python:3.10.12 as builder
+RUN apt-get update && apt-get install -y build-essential \
+    && rm -rf /var/lib/apt/lists/* \
+    && python -m venv /venv \
+    && mkdir /build
+ENV VIRTUAL_ENV=/venv
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+WORKDIR /build
+COPY ./ ./
+RUN "${VIRTUAL_ENV}/bin/pip" install --upgrade pip \
+    && LIBRARY_PATH=/lib:/usr/lib /bin/sh -c "${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements.txt" \
+    && "${VIRTUAL_ENV}/bin/python3" -m nltk.downloader -d /usr/share/nltk_data stopwords
+ARG APP_VERSION
+ARG RELEASE_MODE
+ARG GITHUB_TOKEN
+RUN if [ "$RELEASE_MODE" = "true" ]; then make release v=${APP_VERSION} githubtoken=${GITHUB_TOKEN}; else if [ "${APP_VERSION}" != "" ]; then make build-release v=${APP_VERSION}; fi ; fi
+RUN mkdir /backend \
+    && cp /build/VERSION /backend \
+    && cp -r /build/app /backend/ \
+    && cp -r /build/res /backend/
+
+
+FROM --platform=${BUILDPLATFORM} python:3.10.12-slim
 RUN apt-get update && apt-get -y upgrade \
     && apt-get install -y libxml2 libgomp1 curl \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /venv /venv
+COPY --from=builder /venv /venv
 RUN mkdir /usr/share/nltk_data && chmod g+w /usr/share/nltk_data
-COPY --from=0 /usr/share/nltk_data /usr/share/nltk_data/
-
+COPY --from=builder /usr/share/nltk_data /usr/share/nltk_data/
 WORKDIR /backend/
-
-COPY . .
-COPY --from=0 /backend/VERSION /backend/.bumpversion.cfg ./
-RUN rm -rf /backend/.git/
-
+COPY --from=builder /backend ./
 EXPOSE 5001
-
+ENV VIRTUAL_ENV="/venv"
 # uWSGI configuration (customize as needed):
-ENV FLASK_APP=app.py UWSGI_WSGI_FILE=app.py UWSGI_SOCKET=:3031 UWSGI_HTTP=:5001 UWSGI_VIRTUALENV=/venv UWSGI_MASTER=1 UWSGI_WORKERS=4 UWSGI_THREADS=8 UWSGI_LAZY_APPS=1 UWSGI_WSGI_ENV_BEHAVIOR=holy PYTHONDONTWRITEBYTECODE=1
-ENV PATH="/venv/bin:${PATH}"
-ENV PYTHONPATH="/backend"
-
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" PYTHONPATH=/backend \
+    FLASK_APP=app/main.py UWSGI_WSGI_FILE=app/main.py UWSGI_SOCKET=:3031 UWSGI_HTTP=:5001 \
+    UWSGI_VIRTUALENV=${VIRTUAL_ENV} UWSGI_MASTER=1 UWSGI_WORKERS=4 UWSGI_THREADS=8 UWSGI_LAZY_APPS=1 \
+    UWSGI_WSGI_ENV_BEHAVIOR=holy PYTHONDONTWRITEBYTECODE=1
 # Start uWSGI
 CMD ["/venv/bin/uwsgi", "--http-auto-chunked", "--http-keepalive"]
-HEALTHCHECK --interval=1m --timeout=5s --retries=2 CMD ["curl","-s", "-f", "--show-error","http://localhost:5001/"]
+HEALTHCHECK --interval=1m --timeout=5s --retries=2 CMD ["curl", "-s", "-f", "--show-error", "http://localhost:5001/"]