diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 10b6233e..93a848d0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -8,13 +8,31 @@ on: branches: - main +env: + ACS_CHART_VERSION: 8.3.0 + ARTIFACT_NAME: alfresco-docker-images + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} cancel-in-progress: true +permissions: + contents: read + actions: read + jobs: + pre-commit: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v6.0.0 + with: + auto-commit: "true" + build-test: runs-on: ubuntu-latest + needs: pre-commit steps: - name: Checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -31,27 +49,32 @@ jobs: run: ./scripts/fetch-artifact.sh - name: Set up QEMU - uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v3.1.0 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2ad185228a349d19414702819e06df9fa4314287 # v3.4.0 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 - name: Login to Quay.io - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} + - name: Collect Workflow Telemetry + uses: catchpoint/workflow-telemetry-action@v2 + with: + comment_on_pr: false + - name: Bake Docker images - uses: docker/bake-action@eac74531aa56a9266bfedfe5edb6b851ce2cca2b # v5.4.0 + uses: docker/bake-action@a4d7f0b5b91c14a296d792d4ec53a9db17f02e67 # v5.5.0 - name: Show all built images run: docker images - name: Docker save all baked images whose name include `alfresco` run: | - docker save -o /tmp/alfresco-docker-images.tar $(docker images --format "{{.Repository}}:{{.Tag}}" | grep alfresco) + docker save -o /tmp/${{ env.ARTIFACT_NAME }}.tar $(docker images --format "{{.Repository}}:{{.Tag}}" | grep alfresco) - name: Check disk space run: df -h @@ -60,40 +83,128 @@ jobs: - name: Upload images as artifact uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 with: - name: docker-images - path: /tmp/alfresco-docker-images.tar + name: ${{ env.ARTIFACT_NAME }} + path: /tmp/${{ env.ARTIFACT_NAME }}.tar retention-days: 1 compression-level: 0 compose-test: - name: compose-test ${{ matrix.docker-compose-file }} + name: compose-test needs: build-test runs-on: ubuntu-latest - strategy: - matrix: - docker-compose-file: [docker-compose.yml, docker-compose-components.yml] steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2ad185228a349d19414702819e06df9fa4314287 # v3.4.0 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: path: /tmp - name: docker-images + name: ${{ env.ARTIFACT_NAME }} - name: Load Docker images run: | - ls -1 /tmp/*.tar | xargs --no-run-if-empty -L 1 docker load -i + docker load -i /tmp/${{ env.ARTIFACT_NAME }}.tar docker image ls -a - name: Verify docker-compose - uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v5.34.0 + uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v6.0.0 timeout-minutes: 10 with: compose_pull: false - compose_file_path: docker-compose/${{ matrix.docker-compose-file }} + compose_file_path: test/docker-compose.yml quay_username: ${{ secrets.QUAY_USERNAME }} quay_password: ${{ secrets.QUAY_PASSWORD }} - - uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v5.34.0 + - uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v6.0.0 + + helm-test: + name: helm test + needs: build-test + runs-on: alfrescoPub-ubuntu2204-16G-4CPU + steps: + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: /tmp + name: ${{ env.ARTIFACT_NAME }} + + - name: Setup cluster + uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v5.35.0 + with: + ingress-nginx-ref: controller-v1.8.2 + metrics: "true" + + - name: Load Docker images + run: | + kind load image-archive -n chart-testing /tmp/${{ env.ARTIFACT_NAME }}.tar + + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: "3.15.2" + + - name: Login to Quay.io + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Create registries auth secret + run: >- + kubectl create secret generic regcred + --from-file=.dockerconfigjson=$HOME/.docker/config.json + --type=kubernetes.io/dockerconfigjson + + - name: Fetch upstream values for tests + run: >- + curl -ssfL + https://raw.githubusercontent.com/Alfresco/acs-deployment/v${{ env.ACS_CHART_VERSION }}/test/enterprise-integration-test-values.yaml + -o test/helm/enterprise-integration-test-values.yaml && + ls -l test/helm/enterprise-integration-test-values.yaml && + cat test/helm/enterprise-integration-test-values.yaml + + - name: Helm install + id: helm_install + run: | + helm repo add alfresco https://kubernetes-charts.alfresco.com/stable + helm repo update + helm install acs alfresco/alfresco-content-services \ + --version ${{ env.ACS_CHART_VERSION }} \ + --set global.search.sharedSecret="$(openssl rand -hex 24)" \ + --set global.known_urls=http://localhost \ + --set global.alfrescoRegistryPullSecrets=regcred \ + --values test/helm/enterprise-integration-test-values.yaml \ + --values test/helm/test-overrides.yaml + + - name: Watch Helm deployment + env: + HELM_INSTALL_TIMEOUT: 7m + run: | + kubectl get pods --watch & + KWPID=$! + kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods + kill $KWPID + echo "Waiting for ESC Reindexing job to complete... " + kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing + + - name: Debug cluster status after install + if: always() && steps.helm_install.outcome != 'skipped' + run: | + helm ls --all-namespaces --all + kubectl get all --all-namespaces + kubectl describe pod + + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod diff --git a/.gitignore b/.gitignore index 5d181ca5..78355de2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist *.rpm *.gz *.tgz +test/helm/enterprise-integration-test-values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d712cb92..de34afbe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: # General hooks section - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-yaml args: [--allow-multiple-documents] @@ -12,6 +12,6 @@ repos: args: ["--fix=lf"] - id: end-of-file-fixer - repo: https://github.com/sirosen/check-jsonschema - rev: 0.14.3 + rev: 0.29.1 hooks: - id: check-github-workflows diff --git a/docker-compose/docker-compose-components.yml b/docker-compose/docker-compose-components.yml deleted file mode 100644 index 17817adb..00000000 --- a/docker-compose/docker-compose-components.yml +++ /dev/null @@ -1,198 +0,0 @@ -version: "2" -services: - alfresco: - image: localhost/alfresco-content-repository:latest - mem_limit: 1900m - environment: - JAVA_TOOL_OPTIONS: >- - -Dencryption.keystore.type=JCEKS - -Dencryption.cipherAlgorithm=DESede/CBC/PKCS5Padding - -Dencryption.keyAlgorithm=DESede - -Dencryption.keystore.location=/usr/local/tomcat/shared/classes/alfresco/extension/keystore/keystore - -Dmetadata-keystore.password=mp6yc0UD9e - -Dmetadata-keystore.aliases=metadata - -Dmetadata-keystore.metadata.password=oKIWzVdEdA - -Dmetadata-keystore.metadata.algorithm=DESede - JAVA_OPTS: >- - -Ddb.driver=org.postgresql.Driver - -Ddb.username=alfresco - -Ddb.password=alfresco - -Ddb.url=jdbc:postgresql://postgres:5432/alfresco - -Dindex.subsystem.name=elasticsearch - -Delasticsearch.createIndexIfNotExists=true - -Delasticsearch.host=elasticsearch - -Delasticsearch.port=9200 - -Dshare.host=127.0.0.1 - -Dshare.port=8080 - -Dalfresco.host=localhost - -Dalfresco.port=8080 - -Dcsrf.filter.enabled=false - -Daos.baseUrlOverwrite=http://localhost:8080/alfresco/aos - -Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true" - -Ddeployment.method=DOCKER_COMPOSE - -Dtransform.service.enabled=true - -Dtransform.service.url=http://transform-router:8095 - -Dsfs.url=http://shared-file-store:8099/ - -DlocalTransform.core-aio.url=http://transform-core-aio:8090/ - -Ddsync.service.uris=http://localhost:9090/alfresco - -XX:MinRAMPercentage=50 - -XX:MaxRAMPercentage=80 - healthcheck: - test: - - CMD - - curl - - -f - - http://localhost:8080/alfresco/api/-default-/public/alfresco/versions/1/probes/-ready- - interval: 30s - timeout: 3s - retries: 3 - start_period: 1m - transform-router: - mem_limit: 512m - image: localhost/alfresco-transform-router:latest - environment: - ACTIVEMQ_URL: nio://activemq:61616 - CORE_AIO_URL: http://transform-core-aio:8090 - FILE_STORE_URL: >- - http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file - ports: - - "8095:8095" - depends_on: - - activemq - transform-core-aio: - image: alfresco/alfresco-transform-core-aio:5.1.2 - mem_limit: 1536m - environment: - JAVA_OPTS: >- - -XX:MinRAMPercentage=50 - -XX:MaxRAMPercentage=80 - ACTIVEMQ_URL: nio://activemq:61616 - FILE_STORE_URL: >- - http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file - ports: - - "8090:8090" - depends_on: - - activemq - shared-file-store: - image: localhost/alfresco-shared-file-store:latest - mem_limit: 512m - environment: - scheduler.content.age.millis: 86400000 - scheduler.cleanup.interval: 86400000 - ports: - - "8099:8099" - volumes: - - shared-file-store-volume:/tmp/Alfresco/sfs - share: - image: quay.io/alfresco/alfresco-share:23.2.1 - mem_limit: 1g - environment: - REPO_HOST: "alfresco" - REPO_PORT: "8080" - JAVA_OPTS: >- - -XX:MinRAMPercentage=50 - -XX:MaxRAMPercentage=80 - -Dalfresco.host=localhost - -Dalfresco.port=8080 - -Dalfresco.context=alfresco - -Dalfresco.protocol=http - postgres: - image: postgres:14.4 - mem_limit: 512m - environment: - - POSTGRES_PASSWORD=alfresco - - POSTGRES_USER=alfresco - - POSTGRES_DB=alfresco - command: postgres -c max_connections=300 -c log_min_messages=LOG - ports: - - "5432:5432" - elasticsearch: - image: elasticsearch:7.10.1 - environment: - - xpack.security.enabled=false - - discovery.type=single-node - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - cap_add: - - IPC_LOCK - ports: - - 9200:9200 - - 9300:9300 - activemq: - image: alfresco/alfresco-activemq:5.18-jre17-rockylinux8 - mem_limit: 1g - ports: - - "8161:8161" # Web Console - - "5672:5672" # AMQP - - "61616:61616" # OpenWire - - "61613:61613" # STOMP - digital-workspace: - image: quay.io/alfresco/alfresco-digital-workspace:4.4.1 - mem_limit: 128m - environment: - APP_CONFIG_PROVIDER: "ECM" - APP_CONFIG_AUTH_TYPE: "BASIC" - BASE_PATH: ./ - APP_CONFIG_PLUGIN_PROCESS_SERVICE: "false" - APP_CONFIG_PLUGIN_MICROSOFT_ONLINE: "false" - APP_BASE_SHARE_URL: "http://localhost:8080/workspace/#/preview/s" - control-center: - image: quay.io/alfresco/alfresco-control-center:8.4.1 - mem_limit: 128m - environment: - APP_CONFIG_PROVIDER: "ECM" - APP_CONFIG_AUTH_TYPE: "BASIC" - BASE_PATH: ./ - proxy: - image: alfresco/alfresco-acs-nginx:3.4.2 - mem_limit: 128m - ports: - - "8080:8080" - depends_on: - - digital-workspace - - alfresco - - share - - control-center - sync-service: - image: quay.io/alfresco/service-sync:4.0.1 - mem_limit: 1g - environment: - JAVA_OPTS: >- - -Dsql.db.driver=org.postgresql.Driver - -Dsql.db.url=jdbc:postgresql://postgres:5432/alfresco - -Dsql.db.username=alfresco - -Dsql.db.password=alfresco - -Dmessaging.broker.host=activemq - -Drepo.hostname=alfresco - -Drepo.port=8080 - -Ddw.server.applicationConnectors[0].type=http - -XX:MinRAMPercentage=50 - -XX:MaxRAMPercentage=80 - ports: - - "9090:9090" - live-indexing-metadata: - image: localhost/alfresco-elasticsearch-live-indexing-metadata:latest - environment: - SPRING_ELASTICSEARCH_REST_URIS: http://elastic:9200 - SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616 - live-indexing-path: - image: localhost/alfresco-elasticsearch-live-indexing-path:latest - environment: - SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200 - SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616 - live-indexing-content: - image: localhost/alfresco-elasticsearch-live-indexing-content:latest - environment: - SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200 - SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616 - ALFRESCO_SHAREDFILESTORE_BASEURL: http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file -volumes: - shared-file-store-volume: - driver_opts: - type: tmpfs - device: tmpfs diff --git a/scripts/fetch-artifact.sh b/scripts/fetch-artifact.sh index fa651327..a1017ca4 100755 --- a/scripts/fetch-artifact.sh +++ b/scripts/fetch-artifact.sh @@ -17,9 +17,14 @@ for i in $(find . -name artifacts.json -mindepth 2); do ARTIFACT_GROUP=$(jq -r ".artifacts.acs${INDEX_KEY}[$j].group" $i) ARTIFACT_PATH=$(jq -r ".artifacts.acs${INDEX_KEY}[$j].path" $i) ARTIFACT_BASEURL="https://nexus.alfresco.com/nexus/repository/${ARTIFACT_REPO}" + ARTIFACT_FINAL_PATH="${ARTIFACT_PATH}/${ARTIFACT_NAME}-${ARTIFACT_VERSION}${ARTIFACT_EXT}" + if [ -f "${ARTIFACT_FINAL_PATH}" ]; then + echo "Artifact $ARTIFACT_NAME-$ARTIFACT_VERSION already downloaded, skipping..." + continue + fi echo "Downloading $ARTIFACT_GROUP:$ARTIFACT_NAME $ARTIFACT_VERSION from $ARTIFACT_BASEURL" wget "${ARTIFACT_BASEURL}/${ARTIFACT_GROUP//\./\/}/${ARTIFACT_NAME}/${ARTIFACT_VERSION}/${ARTIFACT_NAME}-${ARTIFACT_VERSION}${ARTIFACT_EXT}" \ - -O ${ARTIFACT_PATH}/${ARTIFACT_NAME}-${ARTIFACT_VERSION}${ARTIFACT_EXT} \ + -O "${ARTIFACT_FINAL_PATH}" \ --no-verbose done done diff --git a/docker-compose/docker-compose.yml b/test/docker-compose.yml similarity index 98% rename from docker-compose/docker-compose.yml rename to test/docker-compose.yml index a1a06440..c166a640 100644 --- a/docker-compose/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,3 +1,4 @@ +# Do not move this file to a folder that is not a first level folder version: "2" services: alfresco: diff --git a/test/helm/test-overrides.yaml b/test/helm/test-overrides.yaml new file mode 100644 index 00000000..23861492 --- /dev/null +++ b/test/helm/test-overrides.yaml @@ -0,0 +1,52 @@ +--- +alfresco-repository: + image: + repository: localhost/alfresco-content-repository + tag: latest +#share: +alfresco-search-enterprise: + liveIndexing: + content: + image: + repository: localhost/alfresco-elasticsearch-live-indexing-content + tag: latest + metadata: + image: + repository: localhost/alfresco-elasticsearch-live-indexing-metadata + tag: latest + path: + image: + repository: localhost/alfresco-elasticsearch-live-indexing-path + tag: latest +alfresco-transform-service: + #pdfrenderer: + imagemagick: + image: + repository: localhost/alfresco-imagemagick + tag: latest + libreoffice: + image: + repository: localhost/alfresco-libreoffice + tag: latest + tika: + image: + repository: localhost/alfresco-tika + tag: latest + #transformmisc: + transformrouter: + image: + repository: localhost/alfresco-transform-router + tag: latest + filestore: + image: + repository: localhost/alfresco-shared-file-store + tag: latest +#alfresco-sync-service: +dtas: + additionalArgs: + - --tb=short + - -k + - not test_api_explorer_reachability + config: + config: + search_timeout_seconds: 360