From e0cc72cc2c4865f3d547ce6940b9da3ca5b3f293 Mon Sep 17 00:00:00 2001 From: Mihir Gune Date: Wed, 20 Nov 2024 17:39:18 -0600 Subject: [PATCH] Contributing Dockerfile and script to run nondex in an automated fashion --- auto-run-dockerized-nondex/Dockerfile | 17 +++++ auto-run-dockerized-nondex/README.md | 74 +++++++++++++++++++ auto-run-dockerized-nondex/entrypoint.sh | 90 ++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 auto-run-dockerized-nondex/Dockerfile create mode 100644 auto-run-dockerized-nondex/README.md create mode 100644 auto-run-dockerized-nondex/entrypoint.sh diff --git a/auto-run-dockerized-nondex/Dockerfile b/auto-run-dockerized-nondex/Dockerfile new file mode 100644 index 00000000..c407660a --- /dev/null +++ b/auto-run-dockerized-nondex/Dockerfile @@ -0,0 +1,17 @@ +# Use a Maven image with a configurable Java version +ARG JAVA_VERSION=17 +FROM maven:3.9.4-eclipse-temurin-${JAVA_VERSION} + +WORKDIR /app + +RUN apt-get update && apt-get install -y git && apt-get clean + +# Thanks - https://github.com/TestingResearchIllinois/idoft/pull/1491 +ENV MAVEN_OPTS="-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + +COPY entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + diff --git a/auto-run-dockerized-nondex/README.md b/auto-run-dockerized-nondex/README.md new file mode 100644 index 00000000..6a973add --- /dev/null +++ b/auto-run-dockerized-nondex/README.md @@ -0,0 +1,74 @@ +# Dockerized Maven NonDex Runner + +This module provides a **generic and standardized Docker-based NonDex execution platform**, to detect flaky tests in Maven based Java projects. + +## Usage + +### Step 1: Build the Docker Image +With the provided `Dockerfile` and `entrypoint.sh` files in your current directory, build the Docker image using: + +```bash +docker build --build-arg JAVA_VERSION=17 -t nondex-runner . +``` +### Step 2: Run the Docker Image +Run the Docker container using the following command (sample below): + +```bash +docker run \ + -e REPO_URL=https://github.com/apache/myfaces-tobago.git \ + -e RUN_NONDEX_WITH_FN=true \ + -e MODULE="tobago-core" \ + -e NONDEX_RUNS=50 \ + -e TESTS="org.apache.myfaces.tobago.webapp.TobagoResponseWriterUnitTest" \ + -v $(pwd)/nondex_logs:/app/nondex_logs \ + nondex-runner + +``` + +### Arguments + +| **Environment Variable** | **Required** | **Description** | +|-----------------------------|--------------|-----------------------------------------------------------------------------------------------------| +| `REPO_URL` | **Yes** | URL of the GitHub repository to clone and test. | +| `RUN_NONDEX_WITH_FN` | No | Runs NonDex with the `--fail-never` flag if set to `true`. Default: `false`. | +| `MODULE` | No | The specific Maven module to target (passed with `-pl`). | +| `NONDEX_RUNS` | No | The number of NonDex runs (`-DnondexRuns=`). | +| `TESTS` | No | Specify a particular test or suite to run using Maven’s `-Dtest` option. | + +--- + +## Script Workflow + +### 1. Clone the Repository +The script clones the repository provided in the `REPO_URL` argument. + +### 2. Maven Clean Install +The script runs: + +```bash +mvn clean install -DskipTests +``` + +If the MODULE argument is provided, it uses: + +```bash +mvn clean install -pl -DskipTests +``` +### 3. Execute Nondex +The script executes + +```bash +mvn edu.illinois:nondex-maven-plugin:2.1.7:nondex +``` + +With additional arguments if applicable: +- `--fail-never` if `RUN_NONDEX_WITH_FN=true`. +- `-pl ` if `MODULE` is provided. +- `-DnondexRuns=` if `NONDEX_RUNS` is set +- `-Dtest=` if `TESTS` is specified. + +### Output + +- NonDex execution logs are written to the file `$(pwd)/nondex_logs/nondex-.log` +- The **final list of flaky tests** is extracted and saved in `$(pwd)/nondex_logs/nondex-flaky-tests-.log` + diff --git a/auto-run-dockerized-nondex/entrypoint.sh b/auto-run-dockerized-nondex/entrypoint.sh new file mode 100644 index 00000000..42909834 --- /dev/null +++ b/auto-run-dockerized-nondex/entrypoint.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -e + +# Checking if the git repo url is set +if [ -z "$REPO_URL" ]; then + echo "Error: REPO_URL is not set. It is a required argument." + exit 1 +fi + +echo "Cloning repository: $REPO_URL" +git clone "$REPO_URL" repo + +cd repo + +echo "Printing java version for reference" +java --version + +# Initialize the Maven install command +MVN_INSTALL_CMD="mvn clean install" + +# Add the -pl option if MODULE is provided +if [ -n "$MODULE" ]; then + MVN_INSTALL_CMD="$MVN_INSTALL_CMD -pl $MODULE" +fi + +# Output and execute the command +echo "Running 'mvn clean install' with command: $MVN_INSTALL_CMD" +eval $MVN_INSTALL_CMD + +# Creating a directory for logs, volume mount to get onto your local +NONDEX_LOG_DIR=/app/nondex_logs +mkdir -p "$NONDEX_LOG_DIR" + +# Building nondex command using the various options configured - eg. mvn -pl edu.illinois:nondex-maven-plugin:2.1.7:nondex -Dtest="" --fail-never +NONDEX_CMD="mvn" + +if [ -n "$MODULE" ]; then + NONDEX_CMD="$NONDEX_CMD -pl $MODULE" +fi + +NONDEX_CMD="$NONDEX_CMD edu.illinois:nondex-maven-plugin:2.1.7:nondex" + +if [ -n "$TESTS" ]; then + NONDEX_CMD="$NONDEX_CMD -Dtest=\"$TESTS\"" +fi + +if [ -n "$NONDEX_RUNS" ]; then + NONDEX_CMD="$NONDEX_CMD -DnondexRuns=$NONDEX_RUNS" +fi + +if [ "$RUN_NONDEX_WITH_FN" == "true" ]; then + NONDEX_CMD="$NONDEX_CMD --fail-never" +fi + +echo "Final NonDex command: $NONDEX_CMD" + +NONDEX_LOG_FILE="$NONDEX_LOG_DIR/nondex-logs-$(date +%Y%m%d-%H%M%S).log" +NONDEX_FAILURE_FILE="$NONDEX_LOG_DIR/nondex-flaky-tests-$(date +%Y%m%d-%H%M%S).txt" + +echo "Running the NonDex plugin. Logs will be saved to $NONDEX_LOG_FILE" +eval $NONDEX_CMD | tee "$NONDEX_LOG_FILE" + +if [ ! -f "$NONDEX_LOG_FILE" ]; then + echo "Error: NonDex log file not found at $NONDEX_LOG_FILE." + echo "Listing all files in $NONDEX_LOG_DIR:" + ls -R /app/nondex_logs + exit 1 +fi + +# Scraping the nondex file to get flaky tests +echo "Processing NonDex log file: $NONDEX_LOG_FILE" + +if grep -q "\[INFO\] Across all seeds:" "$NONDEX_LOG_FILE"; then + # Extract failing tests + awk ' + /\[INFO\] Across all seeds:/ { in_section=1; next } + /\[INFO\] Test results can be found at:/ { in_section=0 } + in_section && /^\[INFO\]/ { print substr($0, index($0, $5)) } + ' "$NONDEX_LOG_FILE" | sort -u > "$NONDEX_FAILURE_FILE" + + if [ -s "$NONDEX_FAILURE_FILE" ]; then + echo "Failing tests have been written to: $NONDEX_FAILURE_FILE" + else + echo "No failing tests were found in the NonDex log file." + rm -f "$NONDEX_FAILURE_FILE" + fi +else + echo "No 'Across all seeds:' section found in the NonDex log file." +fi +