From 02eef834c715223fe2d4b0e9c0c8912b0b6d736f Mon Sep 17 00:00:00 2001 From: Almalexia Date: Wed, 13 Nov 2024 07:57:35 -0500 Subject: [PATCH 01/11] renamed run percentage text file to a more descriptive name --- ...est_run_percentage.txt => CI_Constant_saved_run_percentage.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CI_Latest_run_percentage.txt => CI_Constant_saved_run_percentage.txt (100%) diff --git a/CI_Latest_run_percentage.txt b/CI_Constant_saved_run_percentage.txt similarity index 100% rename from CI_Latest_run_percentage.txt rename to CI_Constant_saved_run_percentage.txt From 7dd599ec8d9c0bf5253559ed82b19915c6365452 Mon Sep 17 00:00:00 2001 From: Almalexia Date: Wed, 13 Nov 2024 08:01:42 -0500 Subject: [PATCH 02/11] added additional script to retrieve more specific results from CI runs --- ashe_scripts/specimin_get_run_results.py | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ashe_scripts/specimin_get_run_results.py diff --git a/ashe_scripts/specimin_get_run_results.py b/ashe_scripts/specimin_get_run_results.py new file mode 100644 index 00000000..4d44d794 --- /dev/null +++ b/ashe_scripts/specimin_get_run_results.py @@ -0,0 +1,60 @@ +import sys +import os +import re + +def print_repo_specifics(file_path: str): + with open(file_path, 'r') as file: + lines: list[str] = file.readlines() + + repo_name: str = "" + branch_name: str = "" + repo_branch_holder = "", "" + for line in lines: + line: str = line.strip() + + repo_branch_holder = __extract_case(line) + if repo_branch_holder != ("", ""): + repo_name = repo_branch_holder[0] + branch_name = repo_branch_holder[1] + + if "Minimizing source file..." in line: + print("'minimization_attempts' " + repo_name + " " + branch_name) + if "BUILD SUCCESSFUL" in line: + print("'successful_minimization' "+repo_name + " " + branch_name) + if "BUILD FAILED" in line: + print("'failed_minimization' "+repo_name + " " + branch_name) + if "Compiling Java files" in line: + print("'compilation_attempts' "+repo_name + " " + branch_name) + if "Minimized files compiled successfully." in line: + print("'successful_compilation' "+repo_name + " " + branch_name) + print("'full_success' "+repo_name + " " + branch_name) + if "Minimized files failed to compile." in line: + print("'failed_compilation' "+repo_name + " " + branch_name) + +def __extract_case(log_line: str): + """ + Extracts the repository path and branch name from a log line. + + Parameters: + - log_line (str): A string from the log file containing repository and branch information. + + Returns: + - tuple: A tuple containing the repository path and the branch name. + """ + # regex pattern to find the repository path and branch name + pattern = r'--targetFile "(.+?\.java)" --targetMethod "(.+?)"' + match = re.search(pattern, log_line) + + if match: + targetFile = match.group(1).strip() + targetMethod = match.group(2).strip() + return targetFile, targetMethod + else: + return "", "" + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: python3 specimin_repo_specifics.py ") + sys.exit(1) + log_file_path = sys.argv[1] + print_repo_specifics(log_file_path) From ac49fbe431672f0192a8a9bd8e2ae3933ddc46ae Mon Sep 17 00:00:00 2001 From: Almalexia Date: Wed, 13 Nov 2024 08:52:49 -0500 Subject: [PATCH 03/11] changed script to write to a file instead of printing --- ashe_scripts/specimin_get_run_results.py | 60 +++++++++++++----------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/ashe_scripts/specimin_get_run_results.py b/ashe_scripts/specimin_get_run_results.py index 4d44d794..7a5d8804 100644 --- a/ashe_scripts/specimin_get_run_results.py +++ b/ashe_scripts/specimin_get_run_results.py @@ -3,37 +3,41 @@ import re def print_repo_specifics(file_path: str): - with open(file_path, 'r') as file: - lines: list[str] = file.readlines() - - repo_name: str = "" - branch_name: str = "" - repo_branch_holder = "", "" - for line in lines: - line: str = line.strip() - - repo_branch_holder = __extract_case(line) - if repo_branch_holder != ("", ""): - repo_name = repo_branch_holder[0] - branch_name = repo_branch_holder[1] - - if "Minimizing source file..." in line: - print("'minimization_attempts' " + repo_name + " " + branch_name) - if "BUILD SUCCESSFUL" in line: - print("'successful_minimization' "+repo_name + " " + branch_name) - if "BUILD FAILED" in line: - print("'failed_minimization' "+repo_name + " " + branch_name) - if "Compiling Java files" in line: - print("'compilation_attempts' "+repo_name + " " + branch_name) - if "Minimized files compiled successfully." in line: - print("'successful_compilation' "+repo_name + " " + branch_name) - print("'full_success' "+repo_name + " " + branch_name) - if "Minimized files failed to compile." in line: - print("'failed_compilation' "+repo_name + " " + branch_name) + directory: str = os.path.dirname(file_path) + output_file_path: str = os.path.join(directory, 'specimin_run_results.txt') + with open(output_file_path, 'a') as output_file: + with open(file_path, 'r') as file: + lines: list[str] = file.readlines() + + repo_name: str = "" + branch_name: str = "" + repo_branch_holder = "", "" + for line in lines: + line: str = line.strip() + + repo_branch_holder = "","" + repo_branch_holder = __extract_case(line) + if repo_branch_holder != ("", ""): + repo_name = repo_branch_holder[0] + branch_name = repo_branch_holder[1] + + if "Minimizing source file..." in line: + output_file.write("'new minimization attempt'\n") + if "BUILD SUCCESSFUL" in line: + output_file.write("'successful_minimization' "+repo_name + " " + branch_name + "\n") + if "BUILD FAILED" in line: + output_file.write("'failed_minimization' "+repo_name + " " + branch_name + "\n") + if "Compiling Java files" in line: + output_file.write("'compilation_attempts' "+repo_name + " " + branch_name + "\n") + if "Minimized files compiled successfully." in line: + output_file.write("'successful_compilation' "+repo_name + " " + branch_name + "\n") + output_file.write("'full_success' "+repo_name + " " + branch_name + "\n") + if "Minimized files failed to compile." in line: + output_file.write("'failed_compilation' "+repo_name + " " + branch_name + "\n") def __extract_case(log_line: str): """ - Extracts the repository path and branch name from a log line. + Extracts the java source file name and method name from a log line. Parameters: - log_line (str): A string from the log file containing repository and branch information. From 8ead20943c573c2707f464d5f197f31fc784b7bb Mon Sep 17 00:00:00 2001 From: Almalexia Date: Wed, 13 Nov 2024 09:36:30 -0500 Subject: [PATCH 04/11] changed variable names in ci script to be more clear, added steps to output resultsto github --- .../workflows/check_compilable_percentage.yml | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index 35d2e7eb..a3a9494a 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -98,34 +98,46 @@ jobs: $(pwd)/CI_repository_list.csv \ $(pwd)/ASHE/CI_REPO_CLONE_SPACE \ $(pwd)/ASHE/src/main/resources/config.properties - + - name: Translate results + run: | + python3 ashe_scripts/specimin_get_run_results.py ASHE/logs/app.log + cat ASHE/logs/app.log + - name: Show only successful minimizations + run: | + grep 'full_success' ASHE/logs/specimin_run_results.txt + - name: Show only failed minimizations + run: | + grep 'failed_minimization' ASHE/logs/specimin_run_results.txt + - name: Show only failed compilations + run: | + grep 'failed_compilation' ASHE/logs/specimin_run_results.txt - name: Parse accuracy percentage id: parse_accuracy_percentage run: | - current_accuracy=$(grep 'Fully successful from minimization to compilation' "$(pwd)/ASHE/logs/specimin_statistics.txt" | awk '{print $NF}' | tr -d '()%') - echo "Current accuracy: $current_accuracy" - echo "::set-output name=current_accuracy::$current_accuracy" + current_run_accuracy=$(grep 'Fully successful from minimization to compilation' "$(pwd)/ASHE/logs/specimin_statistics.txt" | awk '{print $NF}' | tr -d '()%') + echo "Current run accuracy: $current_run_accuracy" + echo "::set-output name=current_run_accuracy::$current_run_accuracy" - name: Read latest run percentage from file - id: read_latest_run_percentage + id: constant_saved_run_percentage run: | - if [ -f "$(pwd)/CI_Latest_run_percentage.txt" ]; then - latest_run_accuracy=$(cat "$(pwd)/CI_Latest_run_percentage.txt" | tr -d '()%') - echo "Latest run accuracy: $latest_run_accuracy" - echo "::set-output name=latest_run_accuracy::$latest_run_accuracy" + if [ -f "$(pwd)/CI_constant_saved_run_percentage" ]; then + constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage" | tr -d '()%') + echo "Constant saved run accuracy: $constant_saved_run_percentage" + echo "::set-output name=constant_saved_run_percentage::$latest_run_accuracy" else - echo "File CI_Latest_run_percentage.txt does not exist" + echo "File CI_constant_saved_run_percentage does not exist" exit 1 fi - name: Validate accuracy id: validate_accuracy run: | - current_accuracy="${{ steps.parse_accuracy_percentage.outputs.current_accuracy }}" - latest_run_accuracy="${{ steps.read_latest_run_percentage.outputs.latest_run_accuracy }}" + current_run_accuracy="${{ steps.parse_accuracy_percentage.outputs.current_run_accuracy }}" + constant_saved_run_percentage="${{ steps.constant_saved_run_percentage.outputs.latest_run_accuracy }}" - if [ "$current_accuracy" != "$latest_run_accuracy" ]; then - echo "Current accuracy ($current_accuracy) does not match latest run accuracy ($latest_run_accuracy)." + if [ "$current_run_accuracy" != "$constant_saved_run_percentage" ]; then + echo "Current accuracy ($current_run_accuracy) does not match latest run accuracy ($constant_saved_run_percentage)." exit 1 else echo "Accuracy validation passed." From 5333c51e0694ccdfce73aac0a3f90666c48736a1 Mon Sep 17 00:00:00 2001 From: Almalexia Date: Wed, 13 Nov 2024 16:05:15 -0500 Subject: [PATCH 05/11] capitalisation fix in CI script reference to text file --- .github/workflows/check_compilable_percentage.yml | 6 +++--- CI_Constant_saved_run_percentage.txt | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 CI_Constant_saved_run_percentage.txt diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index a3a9494a..9326424d 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -121,12 +121,12 @@ jobs: - name: Read latest run percentage from file id: constant_saved_run_percentage run: | - if [ -f "$(pwd)/CI_constant_saved_run_percentage" ]; then - constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage" | tr -d '()%') + if [ -f "$(pwd)/CI_constant_saved_run_percentage.txt" ]; then + constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage.txt" | tr -d '()%') echo "Constant saved run accuracy: $constant_saved_run_percentage" echo "::set-output name=constant_saved_run_percentage::$latest_run_accuracy" else - echo "File CI_constant_saved_run_percentage does not exist" + echo "File CI_constant_saved_run_percentage.txt does not exist" exit 1 fi diff --git a/CI_Constant_saved_run_percentage.txt b/CI_Constant_saved_run_percentage.txt deleted file mode 100644 index 3eced205..00000000 --- a/CI_Constant_saved_run_percentage.txt +++ /dev/null @@ -1 +0,0 @@ -89.06 From d9648f67a788117174b020df37e04bc86d2a17aa Mon Sep 17 00:00:00 2001 From: almalexia Date: Thu, 14 Nov 2024 17:59:54 -0500 Subject: [PATCH 06/11] re-added constant saved run percent text file --- CI_constant_saved_run_percentage.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 CI_constant_saved_run_percentage.txt diff --git a/CI_constant_saved_run_percentage.txt b/CI_constant_saved_run_percentage.txt new file mode 100644 index 00000000..66776207 --- /dev/null +++ b/CI_constant_saved_run_percentage.txt @@ -0,0 +1 @@ +89.01 From 4137d747c655922155de779c037dd17c8b79b73f Mon Sep 17 00:00:00 2001 From: Almalexia Date: Thu, 14 Nov 2024 21:45:46 -0500 Subject: [PATCH 07/11] Forgot to change variable name in CI script. --- .github/workflows/check_compilable_percentage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index 9326424d..b44ec10d 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -134,7 +134,7 @@ jobs: id: validate_accuracy run: | current_run_accuracy="${{ steps.parse_accuracy_percentage.outputs.current_run_accuracy }}" - constant_saved_run_percentage="${{ steps.constant_saved_run_percentage.outputs.latest_run_accuracy }}" + constant_saved_run_percentage="${{ steps.constant_saved_run_percentage.outputs.constant_saved_run_percentage }}" if [ "$current_run_accuracy" != "$constant_saved_run_percentage" ]; then echo "Current accuracy ($current_run_accuracy) does not match latest run accuracy ($constant_saved_run_percentage)." From d79fddf79b9bf8f3fc5dc5098815dc3efaf37c64 Mon Sep 17 00:00:00 2001 From: Almalexia Date: Mon, 18 Nov 2024 23:34:41 -0500 Subject: [PATCH 08/11] corrected bash variable I missed the first time around. --- .github/workflows/check_compilable_percentage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index b44ec10d..93f40cd1 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -124,7 +124,7 @@ jobs: if [ -f "$(pwd)/CI_constant_saved_run_percentage.txt" ]; then constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage.txt" | tr -d '()%') echo "Constant saved run accuracy: $constant_saved_run_percentage" - echo "::set-output name=constant_saved_run_percentage::$latest_run_accuracy" + echo "::set-output name=constant_saved_run_percentage::$constant_saved_run_percentage" else echo "File CI_constant_saved_run_percentage.txt does not exist" exit 1 From 3d9a345957387ba2b37e5385003f3035032ddd52 Mon Sep 17 00:00:00 2001 From: Almalexia Date: Tue, 19 Nov 2024 09:45:00 -0500 Subject: [PATCH 09/11] updated saved accuracy score constant --- CI_constant_saved_run_percentage.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI_constant_saved_run_percentage.txt b/CI_constant_saved_run_percentage.txt index 66776207..08827f86 100644 --- a/CI_constant_saved_run_percentage.txt +++ b/CI_constant_saved_run_percentage.txt @@ -1 +1 @@ -89.01 +89.49 From 53b3f7ee33058fc4a3f681fbe0873b5a1496d6a2 Mon Sep 17 00:00:00 2001 From: almalexia Date: Tue, 26 Nov 2024 11:40:41 -0500 Subject: [PATCH 10/11] resolved merge conflict with PR 378 --- .github/workflows/check_compilable_percentage.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index 93f40cd1..2b4720c7 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -116,7 +116,7 @@ jobs: run: | current_run_accuracy=$(grep 'Fully successful from minimization to compilation' "$(pwd)/ASHE/logs/specimin_statistics.txt" | awk '{print $NF}' | tr -d '()%') echo "Current run accuracy: $current_run_accuracy" - echo "::set-output name=current_run_accuracy::$current_run_accuracy" + echo "current_run_accuracy=$current_run_accuracy" >> "$GITHUB_ENV" - name: Read latest run percentage from file id: constant_saved_run_percentage @@ -124,7 +124,7 @@ jobs: if [ -f "$(pwd)/CI_constant_saved_run_percentage.txt" ]; then constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage.txt" | tr -d '()%') echo "Constant saved run accuracy: $constant_saved_run_percentage" - echo "::set-output name=constant_saved_run_percentage::$constant_saved_run_percentage" + echo "constant_saved_run_percentage=$constant_saved_run_percentage" >> "$GITHUB_ENV" else echo "File CI_constant_saved_run_percentage.txt does not exist" exit 1 @@ -133,8 +133,6 @@ jobs: - name: Validate accuracy id: validate_accuracy run: | - current_run_accuracy="${{ steps.parse_accuracy_percentage.outputs.current_run_accuracy }}" - constant_saved_run_percentage="${{ steps.constant_saved_run_percentage.outputs.constant_saved_run_percentage }}" if [ "$current_run_accuracy" != "$constant_saved_run_percentage" ]; then echo "Current accuracy ($current_run_accuracy) does not match latest run accuracy ($constant_saved_run_percentage)." From ba93fe45ec840b5531a23d90c15e51f767add772 Mon Sep 17 00:00:00 2001 From: almalexia Date: Tue, 26 Nov 2024 12:24:38 -0500 Subject: [PATCH 11/11] re-added steps output reference in validate accuracy step --- .github/workflows/check_compilable_percentage.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_compilable_percentage.yml b/.github/workflows/check_compilable_percentage.yml index 2b4720c7..001205e2 100644 --- a/.github/workflows/check_compilable_percentage.yml +++ b/.github/workflows/check_compilable_percentage.yml @@ -116,7 +116,7 @@ jobs: run: | current_run_accuracy=$(grep 'Fully successful from minimization to compilation' "$(pwd)/ASHE/logs/specimin_statistics.txt" | awk '{print $NF}' | tr -d '()%') echo "Current run accuracy: $current_run_accuracy" - echo "current_run_accuracy=$current_run_accuracy" >> "$GITHUB_ENV" + echo "current_run_accuracy=$current_run_accuracy" >> "$GITHUB_OUTPUT" - name: Read latest run percentage from file id: constant_saved_run_percentage @@ -124,7 +124,7 @@ jobs: if [ -f "$(pwd)/CI_constant_saved_run_percentage.txt" ]; then constant_saved_run_percentage=$(cat "$(pwd)/CI_constant_saved_run_percentage.txt" | tr -d '()%') echo "Constant saved run accuracy: $constant_saved_run_percentage" - echo "constant_saved_run_percentage=$constant_saved_run_percentage" >> "$GITHUB_ENV" + echo "constant_saved_run_percentage=$constant_saved_run_percentage" >> "$GITHUB_OUTPUT" else echo "File CI_constant_saved_run_percentage.txt does not exist" exit 1 @@ -133,6 +133,8 @@ jobs: - name: Validate accuracy id: validate_accuracy run: | + current_run_accuracy="${{ steps.parse_accuracy_percentage.outputs.current_run_accuracy }}" + constant_saved_run_percentage="${{ steps.constant_saved_run_percentage.outputs.constant_saved_run_percentage }}" if [ "$current_run_accuracy" != "$constant_saved_run_percentage" ]; then echo "Current accuracy ($current_run_accuracy) does not match latest run accuracy ($constant_saved_run_percentage)."