diff --git a/.gitignore b/.gitignore index 956f62b..68bec19 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,8 @@ util/00__release/git* util/00__release/json-v2.sh /docs/ py_env -reports/* +reports* /tmp/ /work-in-progress/ -node_modules \ No newline at end of file +node_modules +/util/05__build_docker diff --git a/00__check_prereqs.sh b/00__check_prereqs.sh index c9b3314..18f9253 100755 --- a/00__check_prereqs.sh +++ b/00__check_prereqs.sh @@ -10,7 +10,8 @@ ############################################################################################################## # ------ Do not modify -NEEDED_MIN_FREE_DISK_SPACE_IN_GB=10 +NEEDED_MIN_FREE_DISK_SPACE_IN_GB=20 +RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB=15 ARE_PREREQUISITES_MET=true IS_CONTAINER_ENGINE_CHECKED=false @@ -64,6 +65,15 @@ function check_container_engine() { fi IS_CONTAINER_ENGINE_RUNNING=false fi + + if [[ "${IS_CONTAINER_ENGINE_RUNNING}" == "true" ]]; then + # Check memory limit + CONTAINER_ENGINE_MEM_TOTAL=$(${CONTAINER_ENGINE} info --format '{{.MemTotal}}' | awk '{print $1/1024/1024/1024}') + CONTAINER_ENGINE_MEM_TOTAL_IN_GB=${CONTAINER_ENGINE_MEM_TOTAL%.*} + if ((CONTAINER_ENGINE_MEM_TOTAL_IN_GB < RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB)); then + log_console_warning "Current container engine (${CONTAINER_ENGINE}) memory limit (${CONTAINER_ENGINE_MEM_TOTAL} GB) is beyond the recommended minimum (${RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB} GB). Please increase it to speed up analysis and prevent memory issues." + fi + fi set -e fi IS_CONTAINER_ENGINE_CHECKED=true @@ -95,50 +105,50 @@ function check_container_engine() { mkdir -p "${APP_DIR_IN}" -log_console_step "Check and import present applications (${TARGET_GROUP})" +log_console_step "Check and import present applications (${APP_GROUP})" # Handle "IMPORT_DIR" parameter -SKIP_TARGET_GROUP_ANALYSIS="false" +SKIP_APP_GROUP_ANALYSIS="false" if [[ -n "${IMPORT_DIR}" ]]; then if [[ ! -d "${IMPORT_DIR}" ]]; then log_console_error "The specified application import directory ('${IMPORT_DIR}') does not exist." ARE_PREREQUISITES_MET=false - SKIP_TARGET_GROUP_ANALYSIS=true + SKIP_APP_GROUP_ANALYSIS=true else if [[ "${IMPORT_DIR}" = "${CURRENT_DIR}"* ]]; then log_console_error "The specified application import directory ('${IMPORT_DIR}') must not under the current dir ('${CURRENT_DIR}')." ARE_PREREQUISITES_MET=false - SKIP_TARGET_GROUP_ANALYSIS=true + SKIP_APP_GROUP_ANALYSIS=true else - rm -Rf "${APP_DIR_IN:?}/${TARGET_GROUP}" + rm -Rf "${APP_DIR_IN:?}/${APP_GROUP}" # Detect if it is a single- or multiple-app directory if (find "${IMPORT_DIR}" -mindepth 1 -maxdepth 1 | rev | cut -d '/' -f 1 | rev | grep -q -i -E '^pom.xml$|^readme.md$|^license.md$|^\.git$|^target$|^\.svn$'); then log_console_info "Import a single application" # Single app in the import directory - TARGET_DIR="${APP_DIR_IN}/${TARGET_GROUP}/" + TARGET_DIR="${APP_DIR_IN}/${APP_GROUP}/" else log_console_info "Import multiple applications" # Multiple apps in the import directory TARGET_DIR="${APP_DIR_IN}/" fi mkdir -p "${TARGET_DIR}" - cp -Rfp "${IMPORT_DIR}/../${TARGET_GROUP}" "${TARGET_DIR}." + cp -Rfp "${IMPORT_DIR}/../${APP_GROUP}" "${TARGET_DIR}." fi fi fi -if [[ "${SKIP_TARGET_GROUP_ANALYSIS}" == "false" ]]; then - if [[ "${TARGET_GROUP}" == "" ]]; then +if [[ "${SKIP_APP_GROUP_ANALYSIS}" == "false" ]]; then + if [[ "${APP_GROUP}" == "" ]]; then # All groups selected SELECTED_APP_DIR_IN="${APP_DIR_IN}" DEPTH=2 COUNT_GROUPS=$(find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' \t') else # Group selected - SELECTED_APP_DIR_IN="${APP_DIR_IN}/${TARGET_GROUP}" + SELECTED_APP_DIR_IN="${APP_DIR_IN}/${APP_GROUP}" DEPTH=1 COUNT_GROUPS=1 - if ! find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | grep -q "${TARGET_GROUP}"; then + if ! find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | grep -q "${APP_GROUP}"; then SELECTED_APP_DIR_IN="" fi fi @@ -153,14 +163,14 @@ if [[ "${SKIP_TARGET_GROUP_ANALYSIS}" == "false" ]]; then ARE_PREREQUISITES_MET=false fi else - if [[ -f "${REPORTS_DIR}/list__${TARGET_GROUP}__all_apps.csv" ]]; then - log_console_warning "Applications do not exist locally. Creating mock application group structure under '${APP_DIR_IN}/${TARGET_GROUP}'" - mkdir -p "${APP_DIR_IN}/${TARGET_GROUP}" + if [[ -f "${REPORTS_DIR}/00__Weave/list__all_apps.csv" ]]; then + log_console_warning "Applications do not exist locally. Creating mock application group structure under '${APP_DIR_IN}/${APP_GROUP}'" + mkdir -p "${APP_DIR_IN}/${APP_GROUP}" while read -r APP; do - touch "${APP_DIR_IN}/${TARGET_GROUP}/${APP}" - done < <(cut -d ',' -f 1 "${REPORTS_DIR}/list__${TARGET_GROUP}__all_apps.csv") + touch "${APP_DIR_IN}/${APP_GROUP}/${APP}" + done < <(cut -d ',' -f 1 "${REPORTS_DIR}/00__Weave/list__all_apps.csv") else - log_console_error "Specified application group ('${TARGET_GROUP}') not found. Please use the import option or organize your applications as described in README.md." + log_console_error "Specified application group ('${APP_GROUP}') not found. Please use the import option or organize your applications as described in README.md." ARE_PREREQUISITES_MET=false fi fi @@ -246,9 +256,9 @@ if [[ "${WINDUP_ACTIVE}" == "true" ]]; then ARE_PREREQUISITES_MET=false fi check_container_engine "${CONTAINER_IMAGE_NAME_WINDUP}" "${DIST_DIR}/oci__windup_${WINDUP_VERSION}.img" - if [[ -z "${WINDUP_INCLUDE_PACKAGES_FILE}" && -z "${WINDUP_EXCLUDE_PACKAGES_FILE}" ]]; then - log_console_warning "Windup Analysis is active, but no list of packages to include/exclude has been set. It might take a long time to run." - fi + #if [[ -z "${WINDUP_INCLUDE_PACKAGES_FILE}" && -z "${WINDUP_EXCLUDE_PACKAGES_FILE}" ]]; then + # log_console_warning "Windup Analysis is active, but no list of packages to include/exclude has been set. It might take a long time to run." + #fi fi # 04 diff --git a/00__weave_execution_plan.sh b/00__weave_execution_plan.sh index 37749a5..ac6ccdd 100755 --- a/00__weave_execution_plan.sh +++ b/00__weave_execution_plan.sh @@ -55,10 +55,31 @@ STEP=$(get_step) SEPARATOR=',' export LOG_FILE=${REPORTS_DIR}/${STEP}__Weave.log +export BASE_DIR="${REPORTS_DIR}/00__Weave" + +# All apps +export LIST_ALL="${BASE_DIR}/list__all_apps.txt" +export LIST_ALL_INIT="${BASE_DIR}/list__all_init_apps.txt" +export LANG_MAP_CSV="${BASE_DIR}/list__all_apps.csv" + +# Currently the following application type can be analyzed ... +# -> Java apps decompiled and initially provided as source code apps (zip/folder) +export LIST_JAVA_SRC="${BASE_DIR}/list__java-src.txt" +# -> Java binary apps +export LIST_JAVA_BIN="${BASE_DIR}/list__java-bin.txt" +# -> Java apps initially provided as source code +export LIST_JAVA_SRC_INIT="${BASE_DIR}/list__java-src-init.txt" +# -> JavaScript / Python / C# source code apps +export LIST_JAVASCRIPT="${BASE_DIR}/list__js.txt" +export LIST_PYTHON="${BASE_DIR}/list__python.txt" +export LIST_DOTNET="${BASE_DIR}/list__cs.txt" +# -> Other apps +export LIST_OTHER="${BASE_DIR}/list__other.txt" + function identify() { - APP_FILE_LIST=${1} - APP_SRC_DIR=${2} - APP_NAME=$(basename "${APP_SRC_DIR}") + local -r APP_FILE_LIST=${1} + local -r APP_SRC_DIR=${2} + local -r APP_NAME=$(basename "${APP_SRC_DIR}") if (grep -q '.*\.java$' "${APP_FILE_LIST}"); then echo "${APP_SRC_DIR}" >>"${LIST_JAVA_SRC}" echo "${APP_SRC_DIR}" >>"${LIST_JAVA_SRC_INIT}" @@ -88,71 +109,47 @@ function sort_files() { done } -function weave() { - GROUP=$(basename "${1}") - SRC_DIR="${1}/src" - TMP_DIR="${1}/tmp" - - # Remove tmp folder to avoid side effects - rm -Rf "${TMP_DIR}" - - # All apps - export LIST_ALL=${REPORTS_DIR}/list__${GROUP}__all_apps.txt - export LIST_ALL_INIT=${REPORTS_DIR}/list__${GROUP}__all_init_apps.txt - - export LANG_MAP_CSV=${REPORTS_DIR}/list__${GROUP}__all_apps.csv - - # Currently the following application type can be analyzed ... - # -> Java apps decompiled and initially provided as source code apps (zip/folder) - export LIST_JAVA_SRC=${REPORTS_DIR}/list__${GROUP}__java-src.txt - # -> Java binary apps - export LIST_JAVA_BIN=${REPORTS_DIR}/list__${GROUP}__java-bin.txt - # -> Java apps initially provided as source code - export LIST_JAVA_SRC_INIT=${REPORTS_DIR}/list__${GROUP}__java-src-init.txt - # -> JavaScript / Python / C# source code apps - export LIST_JAVASCRIPT=${REPORTS_DIR}/list__${GROUP}__js.txt - export LIST_PYTHON=${REPORTS_DIR}/list__${GROUP}__python.txt - export LIST_DOTNET=${REPORTS_DIR}/list__${GROUP}__cs.txt - # -> Other apps - export LIST_OTHER=${REPORTS_DIR}/list__${GROUP}__other.txt +function main() { + rm -Rf "${BASE_DIR}" + mkdir -p "${BASE_DIR}" rm -f "${LIST_ALL}" "${LIST_ALL_INIT}" "${LANG_MAP_CSV}" "${LIST_JAVA_SRC}" "${LIST_JAVA_BIN}" "${LIST_JAVA_SRC_INIT}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}" touch "${LIST_ALL}" "${LIST_ALL_INIT}" "${LANG_MAP_CSV}" "${LIST_JAVA_SRC}" "${LIST_JAVA_BIN}" "${LIST_JAVA_SRC_INIT}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}" # Add all java binary applications to ${LIST_JAVA_BIN} - find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhw]ar' >"${LIST_JAVA_BIN}" + find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhw]ar' >"${LIST_JAVA_BIN}" # Add decompiled Java apps to ${LIST_JAVA_SRC} while read -r APP; do APP_NAME=$(basename "${APP}") - echo "${SRC_DIR}/${APP_NAME}" >>"${LIST_JAVA_SRC}" + echo "${APP_GROUP_SRC_DIR}/${APP_NAME}" >>"${LIST_JAVA_SRC}" echo "${APP_NAME}${SEPARATOR}Java" >>"${LANG_MAP_CSV}" echo "${APP}" >>"${LIST_ALL_INIT}" - done < <(find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhws]ar') + done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhws]ar') # Add zipped source code to their category while read -r ARCHIVE; do APP_NAME=$(basename "${ARCHIVE}") APP_FILE_LIST=${ARCHIVE}.lst unzip -Z1 "${ARCHIVE}" >"${APP_FILE_LIST}" - identify "${APP_FILE_LIST}" "${SRC_DIR}/${APP_NAME%.*}" + identify "${APP_FILE_LIST}" "${APP_GROUP_SRC_DIR}/${APP_NAME%.*}" echo "${ARCHIVE}" >>"${LIST_ALL_INIT}" - done < <(find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.zip') + done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.zip') # Add exploded source code directories to their category while read -r DIR; do DIR_NAME=$(basename "${DIR}") APP_FILE_LIST=${DIR}.list find "${DIR}" -type f >"${APP_FILE_LIST}" - identify "${APP_FILE_LIST}" "${SRC_DIR}/${DIR_NAME}" + identify "${APP_FILE_LIST}" "${APP_GROUP_SRC_DIR}/${DIR_NAME}" echo "${DIR}" >>"${LIST_ALL_INIT}" - done < <(find "${1}" -maxdepth 1 -mindepth 1 -type d -not -name 'src') + done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type d -not -name 'src') cat "${LIST_JAVA_SRC}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}" >"${LIST_ALL}" if [[ "${OWASP_ACTIVE}" == "true" ]]; then # List for OWASP DC - export LIST_OWASP_DC=${REPORTS_DIR}/list__${GROUP}__owasp_dc.txt + export LIST_OWASP_DC=${BASE_DIR}/list__owasp_dc.txt rm -f "${LIST_OWASP_DC}" touch "${LIST_OWASP_DC}" cat "${LIST_ALL}" >"${LIST_OWASP_DC}" @@ -172,11 +169,7 @@ function weave() { COUNT_OTHER_APPS=$(count_lines "${LIST_OTHER}") COUNT_ALL_APPS=$(count_lines "${LIST_ALL}") - log_console_info "[${GROUP}] Identified ${COUNT_ALL_APPS} apps: Java (${COUNT_JAVA_APPS} incl. ${COUNT_JAVA_BIN_APPS} binary), C# (${COUNT_DOTNET_APPS}), JavaScript (${COUNT_JS_APPS}), Python (${COUNT_PY_APPS}), Other (${COUNT_OTHER_APPS})" -} - -function main() { - for_each_group weave + log_console_info "[${APP_GROUP}] Identified ${COUNT_ALL_APPS} apps: Java (${COUNT_JAVA_APPS} incl. ${COUNT_JAVA_BIN_APPS} binary), C# (${COUNT_DOTNET_APPS}), JavaScript (${COUNT_JS_APPS}), Python (${COUNT_PY_APPS}), Other (${COUNT_OTHER_APPS})" } main diff --git a/01__fernflower_decompile.sh b/01__fernflower_decompile.sh index b3ddbf1..76906d5 100755 --- a/01__fernflower_decompile.sh +++ b/01__fernflower_decompile.sh @@ -41,8 +41,9 @@ STEP=$(get_step) VERSION="${FERNFLOWER_VERSION}" # List of all archives that have been decompiled -FERNFLOWER_UNPACKED_LIBS_LIST="${REPORTS_DIR}/${STEP}__Fernflower__unpacked_libs.txt" -FERNFLOWER_ALL_LIBS_LIST="${REPORTS_DIR}/${STEP}__Fernflower__all_libs.txt" +BASE_DIR="${REPORTS_DIR}/${STEP}__Fernflower" +FERNFLOWER_UNPACKED_LIBS_LIST="${BASE_DIR}/list__unpacked_libs.txt" +FERNFLOWER_ALL_LIBS_LIST="${BASE_DIR}/list__all_libs.txt" MVNREPOSITORY_BASE_URL="https://mvnrepository.com" export LOG_FILE=${REPORTS_DIR}/${STEP}__Fernflower.log @@ -79,26 +80,19 @@ function unpack() { # Repack applications to a single huge application without sub-WAR/JAR file function unpack_and_decompile() { - APP_DIR_INCOMING="${1}" - GROUP=$(basename "${APP_DIR_INCOMING}") - APP_DIR_SRC="${APP_DIR_INCOMING}/src" - APP_DIR_TMP="${APP_DIR_INCOMING}/tmp" - - rm -Rf "${APP_DIR_SRC}" "${APP_DIR_TMP}" - mkdir -p "${APP_DIR_SRC}" "${REPORTS_DIR}" - - log_analysis_message "group '${GROUP}'" + rm -Rf "${APP_GROUP_SRC_DIR}" "${APP_GROUP_TMP_DIR}" + mkdir -p "${APP_GROUP_SRC_DIR}" "${REPORTS_DIR}" while read -r APP; do - mkdir -p "${APP_DIR_TMP}" + mkdir -p "${APP_GROUP_TMP_DIR}" log_console_step "Preparing '${APP}' ... " - cp "${APP}" "${APP_DIR_TMP}" + cp "${APP}" "${APP_GROUP_TMP_DIR}" APP_NAME=$(basename "${APP}") - while [ "$(find "${APP_DIR_TMP}" -type f -iname '*.war' \ + while [ "$(find "${APP_GROUP_TMP_DIR}" -type f -iname '*.war' \ -o -type f -iname '*.ear' \ -o -type f -iname '*.jar' \ -o -type f -iname '*.rar' \ @@ -108,8 +102,8 @@ function unpack_and_decompile() { while read -r ARCHIVE; do PARENT_DIR_NAME=$(basename "$(dirname "${ARCHIVE}")") - TMP_DIR_NAME=$(basename "${APP_DIR_TMP}") - ARCHIVE_SHORT_NAME="${ARCHIVE:${#APP_DIR_TMP}+1}" + TMP_DIR_NAME=$(basename "${APP_GROUP_TMP_DIR}") + ARCHIVE_SHORT_NAME="${ARCHIVE:${#APP_GROUP_TMP_DIR}+1}" SHA1_LONG=$(sha1sum "${ARCHIVE}") SHA1=$(echo "${SHA1_LONG}" | cut -d' ' -f 1) @@ -237,7 +231,7 @@ function unpack_and_decompile() { mv "${ARCHIVE}" "${ARCHIVE}.ignored_maven" fi fi - done < <(find "${APP_DIR_TMP}" -type f -iname '*.war' \ + done < <(find "${APP_GROUP_TMP_DIR}" -type f -iname '*.war' \ -o -type f -iname '*.ear' \ -o -type f -iname '*.jar' \ -o -type f -iname '*.rar' \ @@ -246,11 +240,11 @@ function unpack_and_decompile() { -o -type f -iname '*.sar') done - APP_SRC="${APP_DIR_SRC}/${APP_NAME}" + APP_SRC="${APP_GROUP_SRC_DIR}/${APP_NAME}" rm -Rf "${APP_SRC}" mkdir -p "${APP_SRC}" - APP_TMP=${APP_DIR_TMP}/${APP_NAME} + APP_TMP=${APP_GROUP_TMP_DIR}/${APP_NAME} APP_TMP_NAME="${APP_TMP%.*}_${APP_TMP##*.}" if [[ "${PRE_ANALYSIS_ACTIVE}" == "false" ]]; then @@ -268,12 +262,12 @@ function unpack_and_decompile() { else log_console_step "Identified '${APP_TMP_NAME}' with a total of $(find "${APP_TMP_NAME}" -name '*.class' | wc -l | tr -d ' ') classes" fi - # Note: Here we do not delete ${APP_DIR_TMP} as it creates some issues with the containerized fernflower execution. + # Note: Here we do not delete ${APP_GROUP_TMP_DIR} as it creates some issues with the containerized fernflower execution. rm -Rf "${APP_TMP}" "${APP_TMP_NAME}" - done <"${REPORTS_DIR}/list__${GROUP}__java-bin.txt" + done <"${REPORTS_DIR}/00__Weave/list__java-bin.txt" - rm -Rf "${APP_DIR_TMP}" + rm -Rf "${APP_GROUP_TMP_DIR}" } @@ -287,24 +281,24 @@ function check_status() { function main() { - if [[ "${DEBUG}" == "true" ]]; then - set -x - exec 6>&1 - FERNFLOWER_LOG_LEVEL="INFO" - else - exec 6>/dev/null - fi + log_tool_info "Fernflower v${VERSION}" if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_FERNFLOWER}") ]]; then + check_debug_mode + if [[ "${DEBUG}" == "true" ]]; then + FERNFLOWER_LOG_LEVEL="INFO" + fi + set +e - log_tool_info "Fernflower v${VERSION}" [[ "${USE_FINDJAR}" == "true" ]] && { check_status "findjar.com" "${FINDJAR_BASE_URL}"; } check_status "Maven Search" "${MAVEN_SEARCH_BASE_URL}" check_status "MVN Repository" "${MVNREPOSITORY_BASE_URL}" - for_each_group unpack_and_decompile + mkdir -p "${BASE_DIR}" + + unpack_and_decompile if [[ "${PRE_ANALYSIS_ACTIVE}" == "true" ]]; then diff --git a/01__unpack_sources.sh b/01__unpack_sources.sh index a9289a5..9cdeafc 100755 --- a/01__unpack_sources.sh +++ b/01__unpack_sources.sh @@ -3,24 +3,19 @@ # SPDX-License-Identifier: Apache-2.0 ############################################################################################################## -# Unzip all source files and copy all file directories to ${APP_DIR_SRC}. +# Unzip all source files and copy all file directories to ${APP_GROUP_SRC_DIR}. ############################################################################################################## STEP=$(get_step) export LOG_FILE=${REPORTS_DIR}/${STEP}__unpack_sources.log function unpack() { - APP_DIR_INCOMING="${1}" - APP_DIR_SRC="${APP_DIR_INCOMING}/src" - GROUP=$(basename "${APP_DIR_INCOMING}") - log_analysis_message "group '${GROUP}'" - - mkdir -p "${APP_DIR_SRC}" "${REPORTS_DIR}" + mkdir -p "${APP_GROUP_SRC_DIR}" "${REPORTS_DIR}" while read -r APP; do APP_NAME=$(basename "${APP}") - APP_DIR="${APP_DIR_SRC}/${APP_NAME%.*}" + APP_DIR="${APP_GROUP_SRC_DIR}/${APP_NAME%.*}" log_console_step "Extracting '${APP_NAME}' to ${APP_DIR} ..." @@ -40,23 +35,17 @@ function unpack() { fi fi - done < <(find "${APP_DIR_INCOMING}" -maxdepth 1 -mindepth 1 -type f -name '*.zip') + done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.zip') while read -r FOLDER; do - cp -Rfp "${FOLDER}" "${APP_DIR_SRC}/." - done < <(find "${APP_DIR_INCOMING}" -maxdepth 1 -mindepth 1 -type d ! -name 'src') + cp -Rfp "${FOLDER}" "${APP_GROUP_SRC_DIR}/." + done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type d ! -name 'src') } function main() { - set +e - if [[ "${DEBUG}" == "true" ]]; then - set -x - exec 6>&1 - else - exec 6>/dev/null - fi - for_each_group unpack + check_debug_mode + unpack set -e } diff --git a/02__csa__01__analysis.sh b/02__csa__01__analysis.sh index 5d1e941..dd3bb2f 100755 --- a/02__csa__01__analysis.sh +++ b/02__csa__01__analysis.sh @@ -19,35 +19,30 @@ DB_DIR_OUT=${REPORTS_DIR}/${STEP}__CSA/db LOG_FILE=${APP_DIR_OUT}.log LAUNCH_UI_SCRIPT=${APP_DIR_OUT}/../launch_csa_ui.sh -# Analyse all applications present in the ${1} directory. +# Analyze all applications present in the ${APP_GROUP_DIR} directory. function analyze() { - - BASE_DIR="${1}" - SUB_DIR_NAME=$(basename "${BASE_DIR}") - log_analysis_message "group '${SUB_DIR_NAME}'" - declare APP_SUB_DIR_IN TYPE EXT # Analyze decompiled apps if existing - if [ -d "${BASE_DIR}/src" ]; then - APP_SUB_DIR_IN="${BASE_DIR}/src" + if [ -d "${APP_GROUP_SRC_DIR}" ]; then + APP_SUB_DIR_IN="${APP_GROUP_SRC_DIR}" TYPE="d" EXT="/" else - ## DOES NOT REALLY WORKS BY PROVIDING APPS DIRECTLY (WRONG GENERATED RESULTS) - APP_SUB_DIR_IN="${BASE_DIR}" + # FIXME: Does not really work by providing apps directly (incorrect results) + APP_SUB_DIR_IN="${APP_GROUP_DIR}" TYPE="f" EXT="" fi - APP_CONF="${APP_DIR_OUT}/conf-${SUB_DIR_NAME}.yml" + APP_CONF="${APP_DIR_OUT}/conf-${APP_GROUP}.yml" # Build the configuration file (${APP_CONF}) - echo "runName: ${SUB_DIR_NAME}" >"${APP_CONF}" + echo "runName: ${APP_GROUP}" >"${APP_CONF}" echo "applications:" >>"${APP_CONF}" while read -r APP; do APP_NAME=$(echo "${APP}" | rev | cut -d'/' -f1 | rev) - APP_SHORT="/apps/${APP##$BASE_DIR}" + APP_SHORT="/apps/${APP##$APP_GROUP_DIR}" cat >>"${APP_CONF}" <>"${LOG_FILE}" 2>&1 + analyze --database-dir="/db" --config-file="/out/conf-${APP_GROUP}.yml" --output-dir="/out" --display-rule-metrics --display-ignored-files --export=csv,html --export-dir="/out/export" --export-file-name="export") >>"${LOG_FILE}" 2>&1 set -e } @@ -97,7 +92,7 @@ function main() { log_tool_info "Cloud Suitability Analyzer (CSA) v${VERSION}" if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_CSA}") ]]; then mkdir -p "${DB_DIR_OUT}" - for_each_group analyze + analyze create_launch_script log_console_success "CSA analysis completed." log_console_info " Log file: '${LOG_FILE}'" diff --git a/02__csa__02__extract.sh b/02__csa__02__extract.sh index e3313a8..d722ed0 100755 --- a/02__csa__02__extract.sh +++ b/02__csa__02__extract.sh @@ -15,30 +15,24 @@ DB_LOCATION="${APP_DIR_OUT}/db/csa.db" MISSING_FILE="${APP_DIR_OUT}/results_missing.csv" RESULT_BAGGER_FILE_NAME=results_extracted_bagger.csv RESULT_BAGGER_FILE="${APP_DIR_OUT}/${RESULT_BAGGER_FILE_NAME}" -RESULT_FILE="${APP_DIR_OUT}/results_extracted.csv" +RESULT_FILE="${APP_DIR_OUT}/_results_extracted.csv" export LOG_FILE=/dev/null function check_missing_apps() { - GROUP_DIR=${1} - GROUP=$(basename "${GROUP_DIR}") - - log_extract_message "group '${GROUP}'" - while read -r FILE; do APP="$(basename "${FILE}")" if [[ -f "${RESULT_BAGGER_FILE}" ]]; then - if ! grep -q "${GROUP}${S}${APP}${S}" "${RESULT_BAGGER_FILE}"; then - echo "${GROUP}${S}${APP}${S}${APP}${S}n/a${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}" >>"${MISSING_FILE}" + if ! grep -q "${APP_GROUP}${S}${APP}${S}" "${RESULT_BAGGER_FILE}"; then + echo "${APP_GROUP}${S}${APP}${S}${APP}${S}n/a${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}" >>"${MISSING_FILE}" fi else - echo "${GROUP}${S}${APP}${S}${APP}${S}n/a${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}" >>"${MISSING_FILE}" + echo "${APP_GROUP}${S}${APP}${S}${APP}${S}n/a${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}${S}" >>"${MISSING_FILE}" fi - done <"${REPORTS_DIR}/list__${GROUP}__all_apps.txt" + done <"${REPORTS_DIR}/00__Weave/list__all_apps.txt" } function main() { - if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_CSA_BAGGER}") ]]; then # Extract results using bagger rm -f "${RESULT_BAGGER_FILE}" @@ -56,7 +50,7 @@ function main() { # Check missing entries rm -f "${MISSING_FILE}" - for_each_group check_missing_apps + check_missing_apps # Add missing entries touch "${RESULT_BAGGER_FILE}" "${MISSING_FILE}" diff --git a/03__windup__01__package_discovery.sh b/03__windup__01__package_discovery.sh index 00d05c9..08579c5 100755 --- a/03__windup__01__package_discovery.sh +++ b/03__windup__01__package_discovery.sh @@ -10,7 +10,6 @@ set -eu # ------ Do not modify - VERSION=${WINDUP_VERSION} STEP=$(get_step) @@ -25,29 +24,20 @@ ALL_KNOWN_PACKAGES_SHORT="_all_known.packages" ALL_KNOWN_PACKAGES="${PACKAGE_DIR_OUT}/${ALL_KNOWN_PACKAGES_SHORT}" TOKEN="XXXXXXXXXXX" JAVA_BIN_APP_FOUND="false" +PACKAGE_FILE="${PACKAGE_DIR_OUT}/${APP_GROUP}.txt" +JAVA_BIN_LIST="${REPORTS_DIR}/00__Weave/list__java-bin.txt" -# Analyze all applications present in the ${1} directory. +# Analyze all applications present in the ${APP_GROUP_DIR} directory. function analyze_packages() { - APP_DIR_INCOMING="${1}" - GROUP=$(basename "${APP_DIR_INCOMING}") - PACKAGE_FILE="${PACKAGE_DIR_OUT}/${GROUP}.txt" - log_analysis_message "group '${GROUP}'" - - # Trick to isolate the analyzed apps from the other folders - DIR_TEMP="${APP_DIR_INCOMING}/tmp" - rm -Rf "${DIR_TEMP}" - - JAVA_BIN_LIST="${REPORTS_DIR}/list__${GROUP}__java-bin.txt" - if [[ -s "${JAVA_BIN_LIST}" ]]; then JAVA_BIN_APP_FOUND="true" - mkdir -p "${DIR_TEMP}" + mkdir -p "${APP_GROUP_TMP_DIR}" while read -r FILE; do - cp -fp "${FILE}" "${DIR_TEMP}/." || true - done <"${REPORTS_DIR}/list__${GROUP}__java-bin.txt" + cp -fp "${FILE}" "${APP_GROUP_TMP_DIR}/." || true + done <"${JAVA_BIN_LIST}" set +e - (time ${CONTAINER_ENGINE} run ${CONTAINER_ENGINE_ARG} --rm -v "${DIR_TEMP}:/apps" --name Windup "${CONTAINER_IMAGE_NAME_WINDUP}" -b --discoverPackages --input "/apps" -d >"${PACKAGE_FILE}") >>"${LOG_FILE}" 2>&1 + (time ${CONTAINER_ENGINE} run ${CONTAINER_ENGINE_ARG} --rm -v "${APP_GROUP_TMP_DIR}:/apps" --name Windup "${CONTAINER_IMAGE_NAME_WINDUP}" -b --discoverPackages --input "/apps" -d >"${PACKAGE_FILE}") >>"${LOG_FILE}" 2>&1 stream_edit '1,/Known Packages:/d' "${PACKAGE_FILE}" @@ -57,9 +47,9 @@ function analyze_packages() { grep -v "org.jboss.weld.environment.se.WeldContainer" /tmp/pack | grep -v "Weld SE container" >"${PACKAGE_FILE}" set -e - log_console_info "Packages of all apps in the '${GROUP}' group: ${PACKAGE_FILE}" + log_console_info "Packages of all apps in the '${APP_GROUP}' group: ${PACKAGE_FILE}" - rm -Rf "${DIR_TEMP}" + rm -Rf "${APP_GROUP_TMP_DIR}" else log_console_warning "No Java binary application found. Skipping package analysis." fi @@ -83,11 +73,12 @@ function extract_packages() { } function main() { - if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_WINDUP}") ]]; then + # Trick to isolate the analyzed apps from the other folders + rm -Rf "${APP_GROUP_TMP_DIR}" "${PACKAGE_DIR_OUT}" mkdir -p "${PACKAGE_DIR_OUT}" - for_each_group analyze_packages + analyze_packages if [[ "${JAVA_BIN_APP_FOUND}" == "true" ]]; then extract_packages "Known Packages:" "${ALL_KNOWN_PACKAGES}" @@ -118,7 +109,6 @@ function main() { else log_console_error "Windup analysis canceled. Container image unavailable: '${CONTAINER_IMAGE_NAME_WINDUP}'" fi - } main diff --git a/03__windup__02__analysis.sh b/03__windup__02__analysis.sh index 265eb78..273f966 100755 --- a/03__windup__02__analysis.sh +++ b/03__windup__02__analysis.sh @@ -26,46 +26,40 @@ STEP=$(get_step) INCLUDE_PACKAGES=() EXCLUDE_PACKAGES=() -LOG_FILE="${REPORTS_DIR}/${STEP}__WINDUP.log" +APP_DIR_OUT="${REPORTS_DIR}/${STEP}__WINDUP" +LOG_FILE="${APP_DIR_OUT}.log" -# Analyse all applications present in the ${1} directory. +# Analyze all applications present in the ${APP_GROUP_DIR} directory. function analyze() { - APP_DIR_INCOMING=${1} - # Temporary folder for the analyzed applications - APP_DIR_IN_TMP=${APP_DIR_INCOMING}/tmp - GROUP=$(basename "${APP_DIR_INCOMING}") - APP_DIR_OUT=${REPORTS_DIR}/${STEP}__WINDUP__${GROUP} - log_analysis_message "group '${GROUP}'" - - rm -Rf "${APP_DIR_IN_TMP}" + rm -Rf "${APP_GROUP_TMP_DIR}" "${APP_DIR_OUT}" # -> Java binary apps - LIST_JAVA_BIN=${REPORTS_DIR}/list__${GROUP}__java-bin.txt + LIST_JAVA_BIN=${REPORTS_DIR}/00__Weave/list__java-bin.txt # -> Java apps initially provided as source code - LIST_JAVA_SRC_INIT=${REPORTS_DIR}/list__${GROUP}__java-src-init.txt + LIST_JAVA_SRC_INIT=${REPORTS_DIR}/00__Weave/list__java-src-init.txt # Windup can deal with compiled and source apps at the same time. # Source code directories need to have a name ending by ".jar" ".war" or ".ear" if [[ -s "${LIST_JAVA_BIN}" || -s "${LIST_JAVA_SRC_INIT}" ]]; then - mkdir -p "${APP_DIR_OUT}" "${APP_DIR_IN_TMP}" + mkdir -p "${APP_DIR_OUT}" "${APP_GROUP_TMP_DIR}" while read -r FILE; do - cp -fp "${FILE}" "${APP_DIR_IN_TMP}/." || true + cp -fp "${FILE}" "${APP_GROUP_TMP_DIR}/." || true done <"${LIST_JAVA_BIN}" # Needed to make sure that the source code directories are visible for Windup while read -r DIR; do BASENAME=$(basename "${DIR}") - cp -Rfp "${DIR}" "${APP_DIR_IN_TMP}/." || true - mv "${APP_DIR_IN_TMP}/${BASENAME}" "${APP_DIR_IN_TMP}/${BASENAME}_SRC.jar" + cp -Rfp "${DIR}" "${APP_GROUP_TMP_DIR}/." || true + mv "${APP_GROUP_TMP_DIR}/${BASENAME}" "${APP_GROUP_TMP_DIR}/${BASENAME}_SRC.jar" done <"${LIST_JAVA_SRC_INIT}" local ARGS=( -b --target "${TARGET}" - --input "/${GROUP}" + --input "/${APP_GROUP}" --output "/cache" --overwrite ) @@ -88,7 +82,7 @@ function analyze() { log_console_info "EXCLUDE: ${EXCLUDE_PACKAGES[*]:-none}" set +e - (time ${CONTAINER_ENGINE} run ${CONTAINER_ENGINE_ARG} --rm -v "${APP_DIR_IN_TMP}:/${GROUP}:ro" -v "${APP_DIR_OUT}:/out:delegated" -v "tmpfs:/cache:delegated" --name Windup "${CONTAINER_IMAGE_NAME_WINDUP}" "${ARGS[@]}") >>"${LOG_FILE}" 2>&1 + (time ${CONTAINER_ENGINE} run ${CONTAINER_ENGINE_ARG} --rm -v "${APP_GROUP_TMP_DIR}:/${APP_GROUP}:ro" -v "${APP_DIR_OUT}:/out:delegated" -v "tmpfs:/cache:delegated" --name Windup "${CONTAINER_IMAGE_NAME_WINDUP}" "${ARGS[@]}") >>"${LOG_FILE}" 2>&1 # Hack to fix the issue with files created with root user if sudo -n ls >/dev/null 2>&1; then @@ -97,7 +91,7 @@ function analyze() { set -e # Cleanup - rm -Rf "${APP_DIR_IN_TMP}" + rm -Rf "${APP_GROUP_TMP_DIR}" if [[ -s "${APP_DIR_OUT}/index.html" ]]; then log_console_success "Results: ${APP_DIR_OUT}/index.html" @@ -126,7 +120,6 @@ function load_packages() { } function main() { - log_tool_info "Windup v${VERSION}" if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_WINDUP}") ]]; then @@ -137,7 +130,7 @@ function main() { load_packages INCLUDE "${INCLUDE_FILES[@]}" # Run Windup - for_each_group analyze + analyze else log_console_error "Windup analysis canceled. Container image unavailable: '${CONTAINER_IMAGE_NAME_WINDUP}'" diff --git a/03__windup__03__extract.sh b/03__windup__03__extract.sh index 49399e5..40447ea 100755 --- a/03__windup__03__extract.sh +++ b/03__windup__03__extract.sh @@ -14,19 +14,17 @@ STEP=$(get_step) LOG_FILE="${REPORTS_DIR}/${STEP}__WINDUP.log" VERSION="${WINDUP_VERSION}" -function check_missing_apps() { - APP_DIR_INCOMING=${1} - GROUP=$(basename "${APP_DIR_INCOMING}") - OUT_GROUP="${REPORTS_DIR}/${STEP}__WINDUP__${GROUP}" - RESULT_SHELL_FILE="${OUT_GROUP}__results_extracted_shell.csv" - MISSING_FILE="${OUT_GROUP}__results_missing.csv" - RESULT_FILE="${OUT_GROUP}__results_extracted.csv" - RESULT_REPORT_MAP="${OUT_GROUP}__report_map.js" - LIST_JAVA_BIN="${REPORTS_DIR}/list__${GROUP}__java-bin.txt" - LIST_JAVA_SRC_INIT="${REPORTS_DIR}/list__${GROUP}__java-src-init.txt" - - log_extract_message "group '${GROUP}'" +RESULT_DIR="${REPORTS_DIR}/${STEP}__WINDUP" +RESULT_SHELL_FILE="${RESULT_DIR}/_results_extracted_shell.csv" +MISSING_FILE="${RESULT_DIR}/_results_missing.csv" +RESULT_FILE="${RESULT_DIR}/_results_extracted.csv" +RESULT_REPORT_MAP="${RESULT_DIR}/_report_map.js" +LIST_JAVA_BIN="${REPORTS_DIR}/00__Weave/list__java-bin.txt" +LIST_JAVA_SRC_INIT="${REPORTS_DIR}/00__Weave/list__java-src-init.txt" +LIST_ALL_APPS="${REPORTS_DIR}/00__Weave/list__all_apps.txt" +XSL_FILE="${CURRENT_DIR}/conf/Windup/process_WINDUP.xsl" +function check_missing_apps() { rm -f "${MISSING_FILE}" "${RESULT_REPORT_MAP}" # Add missing entries @@ -41,7 +39,7 @@ function check_missing_apps() { else echo "${APP}${SEPARATOR}n/a" >>"${MISSING_FILE}" fi - done <"${REPORTS_DIR}/list__${GROUP}__all_apps.txt" + done <"${LIST_ALL_APPS}" # Merge results with missing entries touch "${RESULT_SHELL_FILE}" "${MISSING_FILE}" @@ -60,7 +58,7 @@ function check_missing_apps() { mv "${RESULT_FILE}.tmp" "${RESULT_FILE}" # Generate HTML report map - WINDUP_REPORT_DIR="${REPORTS_DIR}/${STEP}__WINDUP__${GROUP}/reports/" + WINDUP_REPORT_DIR="${REPORTS_DIR}/${STEP}__WINDUP/reports/" if [[ ! -d "${WINDUP_REPORT_DIR}" ]]; then if [[ -s "${LIST_JAVA_BIN}" || -s "${LIST_JAVA_SRC_INIT}" ]]; then log_console_error "WINDUP result folder does not exist: ${WINDUP_REPORT_DIR}" @@ -87,7 +85,7 @@ function check_missing_apps() { fi REPORT_NAME="$(basename "${REPORT_FULL_NAME}")" echo " ['${APP}', '${REPORT_NAME}']," >>"${RESULT_REPORT_MAP_TMP}" - done <"${REPORTS_DIR}/list__${GROUP}__all_apps.txt" + done <"${LIST_ALL_APPS}" { echo "let reportMap = new Map([" @@ -99,46 +97,35 @@ function check_missing_apps() { } function main() { - - XSL_FILE="${CURRENT_DIR}/conf/Windup/process_WINDUP.xsl" - - #set -x - while read -r FILE; do - REGEX="s|${STEP}__WINDUP__(.*)|\1|g" - GROUP=$(basename "$(dirname "${FILE}")" | sed -Ee "${REGEX}") - GROUP_DIR=${REPORTS_DIR}/${STEP}__WINDUP__${GROUP} - RESULT_FILE="${GROUP_DIR}__results_extracted_shell.csv" - # "2>/dev/null" hides all parsing errors from the console - xmllint --html --xmlout "${FILE}" 2>/dev/null | xsltproc --stringparam separator "${SEPARATOR}" "${XSL_FILE}" - >>"${RESULT_FILE}" + xmllint --html --xmlout "${FILE}" 2>/dev/null | xsltproc --stringparam separator "${SEPARATOR}" "${XSL_FILE}" - >>"${RESULT_SHELL_FILE}" # Removing dots in numbers - awk 'BEGIN {OFS=FS=","} {$1;gsub(/\./,"",$2)}1' "${RESULT_FILE}" >"${RESULT_FILE}.tmp" + awk 'BEGIN {OFS=FS=","} {$1;gsub(/\./,"",$2)}1' "${RESULT_SHELL_FILE}" >"${RESULT_SHELL_FILE}.tmp" # Removing ? in numbers - awk 'BEGIN {OFS=FS=","} {$1;gsub(/\?/,"",$2)}1' "${RESULT_FILE}.tmp" >"${RESULT_FILE}" - rm -f "${RESULT_FILE}.tmp" + awk 'BEGIN {OFS=FS=","} {$1;gsub(/\?/,"",$2)}1' "${RESULT_SHELL_FILE}.tmp" >"${RESULT_SHELL_FILE}" + rm -f "${RESULT_SHELL_FILE}.tmp" # Cleaning up the generated windup report if [[ "${IS_MAC}" == "true" ]]; then ## Removing trackers slowing down pages - find "${GROUP_DIR}" -maxdepth 2 -name "*.html" -type f -exec sed -i '' '/