From 13bb4cfd1005d8e62885c91288b53df046404778 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Mon, 3 Feb 2025 13:10:52 -0800
Subject: [PATCH 1/7] RANGER-5126: Add coverage for testing ranger upgrades
---
.github/workflows/test-upgrade.yaml | 211 ++++++++++++++++++++++++++++
1 file changed, 211 insertions(+)
create mode 100644 .github/workflows/test-upgrade.yaml
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
new file mode 100644
index 0000000000..93af49bdc9
--- /dev/null
+++ b/.github/workflows/test-upgrade.yaml
@@ -0,0 +1,211 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upgrade Ranger
+
+on:
+ push:
+ pull_request:
+ branches: [ "master" ]
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+ install:
+ strategy:
+ fail-fast: false # Ensures all matrix jobs run to completion
+ matrix:
+ release: [ranger-2.4, ranger-2.5, ranger-2.6] # builds all these release versions
+ runs-on: ubuntu-22.04
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v4
+ - name: Cache for maven dependencies
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/ranger
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-
+
+ - name: Set up .m2 directory and permissions
+ run: |
+ mkdir -p ~/.m2/repository
+ chmod -R 777 ~/.m2
+ chmod -R 777 ./
+
+ - name: Set up JDK 8
+ uses: actions/setup-java@v4
+ with:
+ java-version: '8'
+ distribution: 'temurin'
+
+ - name: Update .env
+ run: |
+ cd dev-support/ranger-docker
+ RELEASE="${{ matrix.release }}"
+ python3 <<EOF
+ branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
+ with open(r'.env', 'r') as file:
+ data = file.read()
+ release_branch = "$RELEASE"
+ data = data.replace('BUILD_HOST_SRC=true', 'BUILD_HOST_SRC=false')
+ data = data.replace('BRANCH=master', f'BRANCH={release_branch}')
+ ranger_version = branch_to_version[release_branch]
+ data = data.replace('3.0.0-SNAPSHOT', ranger_version)
+ with open(r'.env', 'w') as file:
+ file.write(data)
+ EOF
+ cat .env
+
+ - name: Run download-archives.sh
+ run: |
+ cd dev-support/ranger-docker
+ ./download-archives.sh none
+
+ - name: Clean up Docker space
+ run: docker system prune --all --force --volumes
+
+ - name: Build Ranger in Docker
+ run: |
+ cd dev-support/ranger-docker
+ export DOCKER_BUILDKIT=1
+ export COMPOSE_DOCKER_CLI_BUILD=1
+ docker compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml build
+ docker compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up -d
+
+ # Get container ID of ranger-build
+ CONTAINER_ID=$(docker ps -aqf "name=ranger-build")
+
+ docker logs -f $CONTAINER_ID &
+
+ # Wait for the container to exit
+ docker wait $CONTAINER_ID
+
+ - name: Upload build artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ matrix.release }}-build-artifacts
+ path: dev-support/ranger-docker/dist/*
+
+ upgrade:
+ needs: install
+ strategy:
+ fail-fast: false
+ matrix:
+ release: [ranger-2.4, ranger-2.5]
+ db: [postgres, mysql]
+ runs-on: ubuntu-22.04
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK 8
+ uses: actions/setup-java@v4
+ with:
+ java-version: '8'
+ distribution: 'temurin'
+
+ - name: Download release artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ matrix.release }}-build-artifacts
+
+ - name: Download release artifacts for upgrade
+ uses: actions/download-artifact@v4
+ with:
+ name: ranger-2.6-build-artifacts
+
+ - name: Copy artifacts for docker build
+ run: |
+ cp ranger-*.tar.gz dev-support/ranger-docker/dist
+ cp version dev-support/ranger-docker/dist
+ ls -lrt dev-support/ranger-docker/dist/
+
+ - name: Update .env
+ run: |
+ cd dev-support/ranger-docker
+ RELEASE="${{ matrix.release }}"
+ python3 <<EOF
+ branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
+ with open(r'.env', 'r') as file:
+ data = file.read()
+ release_branch = "$RELEASE"
+ ranger_version = branch_to_version[release_branch]
+ data = data.replace('3.0.0-SNAPSHOT', ranger_version)
+ with open(r'.env', 'w') as file:
+ file.write(data)
+ EOF
+ cat .env
+
+ - name: Run download-archives.sh
+ run: |
+ cd dev-support/ranger-docker
+ ./download-archives.sh none
+
+ - name: Bringing up ${{ matrix.release }} in docker
+ run: |
+ cd dev-support/ranger-docker
+ export DOCKER_BUILDKIT=1
+ export COMPOSE_DOCKER_CLI_BUILD=1
+ export RANGER_DB_TYPE=${{ matrix.db }}
+ docker compose -f docker-compose.ranger-base.yml build
+ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build
+ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d
+ sleep 30
+ docker logs ranger
+
+ - name: Update .env for upgrade
+ run: |
+ cd dev-support/ranger-docker
+ RELEASE="${{ matrix.release }}"
+ python3 <<EOF
+ branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
+ with open(r'.env', 'r') as file:
+ data = file.read()
+ release_branch = "$RELEASE"
+ ranger_version = branch_to_version[release_branch]
+ data = data.replace(ranger_version, branch_to_version["ranger-2.6"])
+ with open(r'.env', 'w') as file:
+ file.write(data)
+ EOF
+ cat .env
+
+ - name: Upgrading to ranger-2.6 in docker
+ run: |
+ cd dev-support/ranger-docker
+ export DOCKER_BUILDKIT=1
+ export COMPOSE_DOCKER_CLI_BUILD=1
+ export RANGER_DB_TYPE=${{ matrix.db }}
+ docker compose -f docker-compose.ranger-base.yml build
+ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build
+ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d
+ sleep 60
+ docker logs ranger
+
+ - name: Remove running containers
+ run: |
+ docker stop $(docker ps -q) && docker rm $(docker ps -aq)
From 35fc982866b542e96ca118d6fd1a854e3f61012a Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Mon, 3 Feb 2025 15:10:03 -0800
Subject: [PATCH 2/7] Add oracle and refactor file.
---
.github/workflows/test-upgrade.yaml | 18 ++----------------
1 file changed, 2 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
index 93af49bdc9..f9e9f2f8d3 100644
--- a/.github/workflows/test-upgrade.yaml
+++ b/.github/workflows/test-upgrade.yaml
@@ -13,15 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
name: Upgrade Ranger
on:
@@ -82,11 +73,6 @@ jobs:
EOF
cat .env
- - name: Run download-archives.sh
- run: |
- cd dev-support/ranger-docker
- ./download-archives.sh none
-
- name: Clean up Docker space
run: docker system prune --all --force --volumes
@@ -118,7 +104,7 @@ jobs:
fail-fast: false
matrix:
release: [ranger-2.4, ranger-2.5]
- db: [postgres, mysql]
+ db: [postgres, mysql, oracle]
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
@@ -203,7 +189,7 @@ jobs:
docker compose -f docker-compose.ranger-base.yml build
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d
- sleep 60
+ sleep 120
docker logs ranger
- name: Remove running containers
From 4e0b444d42895e2b0d39cc97a9ea938fde043157 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Mon, 3 Feb 2025 17:52:32 -0800
Subject: [PATCH 3/7] Refactor install to build and increase sleep time before
docker logs.
---
.github/workflows/test-upgrade.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
index f9e9f2f8d3..ac937ac662 100644
--- a/.github/workflows/test-upgrade.yaml
+++ b/.github/workflows/test-upgrade.yaml
@@ -24,7 +24,7 @@ env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
jobs:
- install:
+ build:
strategy:
fail-fast: false # Ensures all matrix jobs run to completion
matrix:
@@ -99,7 +99,7 @@ jobs:
path: dev-support/ranger-docker/dist/*
upgrade:
- needs: install
+ needs: build
strategy:
fail-fast: false
matrix:
@@ -189,7 +189,7 @@ jobs:
docker compose -f docker-compose.ranger-base.yml build
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d
- sleep 120
+ sleep 180
docker logs ranger
- name: Remove running containers
From 19c7febe393da66bfbc4702d309b3de5b3e9fc16 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Tue, 4 Feb 2025 18:46:27 -0800
Subject: [PATCH 4/7] Create dot_env_util.py
---
.github/scripts/dot_env_util.py | 58 +++++++++++++++++++++++++++++++++
1 file changed, 58 insertions(+)
create mode 100644 .github/scripts/dot_env_util.py
diff --git a/.github/scripts/dot_env_util.py b/.github/scripts/dot_env_util.py
new file mode 100644
index 0000000000..f25cc6c0a6
--- /dev/null
+++ b/.github/scripts/dot_env_util.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+
+branch_to_snapshot_version_map = {
+ 'master': '3.0.0-SNAPSHOT',
+ 'ranger-2.4': '2.4.1-SNAPSHOT',
+ 'ranger-2.5': '2.5.1-SNAPSHOT',
+ 'ranger-2.6': '2.6.0',
+}
+
+
+def update_env_mvn_build(data, release_branch):
+ data = data.replace('BUILD_HOST_SRC=true', 'BUILD_HOST_SRC=false')
+ data = data.replace('BRANCH=master', f'BRANCH={release_branch}')
+ return data
+
+
+def update_env(current_branch, release_branch, is_mvn_build):
+ with open(r'.env', 'r') as file:
+ data = file.read()
+
+ ranger_release_version = branch_to_snapshot_version_map[release_branch]
+ ranger_current_version = branch_to_snapshot_version_map[current_branch]
+
+ if is_mvn_build:
+ data = update_env_mvn_build(data, release_branch)
+
+ data = data.replace(ranger_current_version, ranger_release_version)
+
+ with open(r'.env', 'w') as file:
+ file.write(data)
+ return
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Utility to update .env')
+ parser.add_argument('--release_branch', help='release branch')
+ parser.add_argument('--current_branch', default='master', help='current checked out ranger branch')
+ parser.add_argument('--maven_build', default=False, help='Maven build in Docker required?')
+
+ args = parser.parse_args()
+ update_env(args.current_branch, args.release_branch, args.maven_build)
+
From dc28730f398158c0b5bc81a5e5e6f1df8ca5a771 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Tue, 4 Feb 2025 18:56:34 -0800
Subject: [PATCH 5/7] Update test-upgrade.yaml
---
.github/workflows/test-upgrade.yaml | 38 +++--------------------------
1 file changed, 3 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
index ac937ac662..72f3be130e 100644
--- a/.github/workflows/test-upgrade.yaml
+++ b/.github/workflows/test-upgrade.yaml
@@ -58,19 +58,7 @@ jobs:
- name: Update .env
run: |
cd dev-support/ranger-docker
- RELEASE="${{ matrix.release }}"
- python3 <<EOF
- branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
- with open(r'.env', 'r') as file:
- data = file.read()
- release_branch = "$RELEASE"
- data = data.replace('BUILD_HOST_SRC=true', 'BUILD_HOST_SRC=false')
- data = data.replace('BRANCH=master', f'BRANCH={release_branch}')
- ranger_version = branch_to_version[release_branch]
- data = data.replace('3.0.0-SNAPSHOT', ranger_version)
- with open(r'.env', 'w') as file:
- file.write(data)
- EOF
+ python3 ./../../.github/scripts/dot_env_util.py --release_branch="${{ matrix.release }}" --current_branch=master --maven_build=True
cat .env
- name: Clean up Docker space
@@ -134,17 +122,7 @@ jobs:
- name: Update .env
run: |
cd dev-support/ranger-docker
- RELEASE="${{ matrix.release }}"
- python3 <<EOF
- branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
- with open(r'.env', 'r') as file:
- data = file.read()
- release_branch = "$RELEASE"
- ranger_version = branch_to_version[release_branch]
- data = data.replace('3.0.0-SNAPSHOT', ranger_version)
- with open(r'.env', 'w') as file:
- file.write(data)
- EOF
+ python3 ./../../.github/scripts/dot_env_util.py --release_branch="${{ matrix.release }}" --current_branch=master
cat .env
- name: Run download-archives.sh
@@ -167,17 +145,7 @@ jobs:
- name: Update .env for upgrade
run: |
cd dev-support/ranger-docker
- RELEASE="${{ matrix.release }}"
- python3 <<EOF
- branch_to_version = {"ranger-2.4": "2.4.1-SNAPSHOT", "ranger-2.5": "2.5.1-SNAPSHOT", "ranger-2.6": "2.6.0"}
- with open(r'.env', 'r') as file:
- data = file.read()
- release_branch = "$RELEASE"
- ranger_version = branch_to_version[release_branch]
- data = data.replace(ranger_version, branch_to_version["ranger-2.6"])
- with open(r'.env', 'w') as file:
- file.write(data)
- EOF
+ python3 ./../../.github/scripts/dot_env_util.py --release_branch=ranger-2.6 --current_branch=master
cat .env
- name: Upgrading to ranger-2.6 in docker
From 6fe57d604480d79bbbc7cfba45a28a0b180176fb Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Tue, 4 Feb 2025 19:07:20 -0800
Subject: [PATCH 6/7] Update test-upgrade.yaml
---
.github/workflows/test-upgrade.yaml | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
index 72f3be130e..42f246cb5b 100644
--- a/.github/workflows/test-upgrade.yaml
+++ b/.github/workflows/test-upgrade.yaml
@@ -16,9 +16,11 @@
name: Upgrade Ranger
on:
- push:
- pull_request:
- branches: [ "master" ]
+ workflow_dispatch:
+ inputs:
+ release_branch:
+ description: 'release branch to be used for upgrade'
+ required: true
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
@@ -28,7 +30,7 @@ jobs:
strategy:
fail-fast: false # Ensures all matrix jobs run to completion
matrix:
- release: [ranger-2.4, ranger-2.5, ranger-2.6] # builds all these release versions
+ release: [ranger-2.4, ranger-2.5, "${{ github.event.inputs.release_branch }}"] # builds all these release versions
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
@@ -111,7 +113,7 @@ jobs:
- name: Download release artifacts for upgrade
uses: actions/download-artifact@v4
with:
- name: ranger-2.6-build-artifacts
+ name: ${{ github.event.inputs.release_branch }}-build-artifacts
- name: Copy artifacts for docker build
run: |
@@ -145,10 +147,10 @@ jobs:
- name: Update .env for upgrade
run: |
cd dev-support/ranger-docker
- python3 ./../../.github/scripts/dot_env_util.py --release_branch=ranger-2.6 --current_branch=master
+ python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch=master
cat .env
- - name: Upgrading to ranger-2.6 in docker
+ - name: Upgrading to ${{ github.event.inputs.release_branch }} in docker
run: |
cd dev-support/ranger-docker
export DOCKER_BUILDKIT=1
From 3d7fbc244c5f07b3f4cc55df98930b4ba1425b13 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishekkumar100031@gmail.com>
Date: Tue, 4 Feb 2025 20:18:16 -0800
Subject: [PATCH 7/7] Fix last update to .env before upgrade
---
.github/workflows/test-upgrade.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml
index 42f246cb5b..acd615ed83 100644
--- a/.github/workflows/test-upgrade.yaml
+++ b/.github/workflows/test-upgrade.yaml
@@ -147,7 +147,7 @@ jobs:
- name: Update .env for upgrade
run: |
cd dev-support/ranger-docker
- python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch=master
+ python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch="${{ matrix.release }}"
cat .env
- name: Upgrading to ${{ github.event.inputs.release_branch }} in docker