Merge pull request #14 from vmware-tanzu/single_group

Single group
vmware-tanzu · Apr 11, 2024 · 92ee358 · 92ee358
2 parents 6c70076 + d500579
commit 92ee358
Show file tree

Hide file tree

Showing 88 changed files with 985 additions and 1,345 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,7 +31,8 @@ util/00__release/git*
 util/00__release/json-v2.sh
 /docs/
 py_env
-reports/*
+reports*
 /tmp/
 /work-in-progress/
-node_modules
+node_modules
+/util/05__build_docker
diff --git a/00__check_prereqs.sh b/00__check_prereqs.sh
@@ -10,7 +10,8 @@
 ##############################################################################################################
 
 # ------ Do not modify
-NEEDED_MIN_FREE_DISK_SPACE_IN_GB=10
+NEEDED_MIN_FREE_DISK_SPACE_IN_GB=20
+RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB=15
 
 ARE_PREREQUISITES_MET=true
 IS_CONTAINER_ENGINE_CHECKED=false
@@ -64,6 +65,15 @@ function check_container_engine() {
 				fi
 				IS_CONTAINER_ENGINE_RUNNING=false
 			fi
+
+			if [[ "${IS_CONTAINER_ENGINE_RUNNING}" == "true" ]]; then
+				# Check memory limit
+				CONTAINER_ENGINE_MEM_TOTAL=$(${CONTAINER_ENGINE} info --format '{{.MemTotal}}' | awk '{print $1/1024/1024/1024}')
+				CONTAINER_ENGINE_MEM_TOTAL_IN_GB=${CONTAINER_ENGINE_MEM_TOTAL%.*}
+				if ((CONTAINER_ENGINE_MEM_TOTAL_IN_GB < RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB)); then
+					log_console_warning "Current container engine (${CONTAINER_ENGINE}) memory limit (${CONTAINER_ENGINE_MEM_TOTAL} GB) is beyond the recommended minimum (${RECOMMENDED_MIN_CONTAINER_ENGINE_MEMORY_IN_GB} GB). Please increase it to speed up analysis and prevent memory issues."
+				fi
+			fi
 			set -e
 		fi
 		IS_CONTAINER_ENGINE_CHECKED=true
@@ -95,50 +105,50 @@ function check_container_engine() {
 
 mkdir -p "${APP_DIR_IN}"
 
-log_console_step "Check and import present applications (${TARGET_GROUP})"
+log_console_step "Check and import present applications (${APP_GROUP})"
 
 # Handle "IMPORT_DIR" parameter
-SKIP_TARGET_GROUP_ANALYSIS="false"
+SKIP_APP_GROUP_ANALYSIS="false"
 if [[ -n "${IMPORT_DIR}" ]]; then
 	if [[ ! -d "${IMPORT_DIR}" ]]; then
 		log_console_error "The specified application import directory ('${IMPORT_DIR}') does not exist."
 		ARE_PREREQUISITES_MET=false
-		SKIP_TARGET_GROUP_ANALYSIS=true
+		SKIP_APP_GROUP_ANALYSIS=true
 	else
 		if [[ "${IMPORT_DIR}" = "${CURRENT_DIR}"* ]]; then
 			log_console_error "The specified application import directory ('${IMPORT_DIR}') must not under the current dir ('${CURRENT_DIR}')."
 			ARE_PREREQUISITES_MET=false
-			SKIP_TARGET_GROUP_ANALYSIS=true
+			SKIP_APP_GROUP_ANALYSIS=true
 		else
-			rm -Rf "${APP_DIR_IN:?}/${TARGET_GROUP}"
+			rm -Rf "${APP_DIR_IN:?}/${APP_GROUP}"
 			# Detect if it is a single- or multiple-app directory
 			if (find "${IMPORT_DIR}" -mindepth 1 -maxdepth 1 | rev | cut -d '/' -f 1 | rev | grep -q -i -E '^pom.xml$|^readme.md$|^license.md$|^\.git$|^target$|^\.svn$'); then
 				log_console_info "Import a single application"
 				# Single app in the import directory
-				TARGET_DIR="${APP_DIR_IN}/${TARGET_GROUP}/"
+				TARGET_DIR="${APP_DIR_IN}/${APP_GROUP}/"
 			else
 				log_console_info "Import multiple applications"
 				# Multiple apps in the import directory
 				TARGET_DIR="${APP_DIR_IN}/"
 			fi
 			mkdir -p "${TARGET_DIR}"
-			cp -Rfp "${IMPORT_DIR}/../${TARGET_GROUP}" "${TARGET_DIR}."
+			cp -Rfp "${IMPORT_DIR}/../${APP_GROUP}" "${TARGET_DIR}."
 		fi
 	fi
 fi
 
-if [[ "${SKIP_TARGET_GROUP_ANALYSIS}" == "false" ]]; then
-	if [[ "${TARGET_GROUP}" == "" ]]; then
+if [[ "${SKIP_APP_GROUP_ANALYSIS}" == "false" ]]; then
+	if [[ "${APP_GROUP}" == "" ]]; then
 		# All groups selected
 		SELECTED_APP_DIR_IN="${APP_DIR_IN}"
 		DEPTH=2
 		COUNT_GROUPS=$(find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | wc -l | tr -d ' \t')
 	else
 		# Group selected
-		SELECTED_APP_DIR_IN="${APP_DIR_IN}/${TARGET_GROUP}"
+		SELECTED_APP_DIR_IN="${APP_DIR_IN}/${APP_GROUP}"
 		DEPTH=1
 		COUNT_GROUPS=1
-		if ! find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | grep -q "${TARGET_GROUP}"; then
+		if ! find "${APP_DIR_IN}" -maxdepth 1 -mindepth 1 -type d | grep -q "${APP_GROUP}"; then
 			SELECTED_APP_DIR_IN=""
 		fi
 	fi
@@ -153,14 +163,14 @@ if [[ "${SKIP_TARGET_GROUP_ANALYSIS}" == "false" ]]; then
 			ARE_PREREQUISITES_MET=false
 		fi
 	else
-		if [[ -f "${REPORTS_DIR}/list__${TARGET_GROUP}__all_apps.csv" ]]; then
-			log_console_warning "Applications do not exist locally. Creating mock application group structure under '${APP_DIR_IN}/${TARGET_GROUP}'"
-			mkdir -p "${APP_DIR_IN}/${TARGET_GROUP}"
+		if [[ -f "${REPORTS_DIR}/00__Weave/list__all_apps.csv" ]]; then
+			log_console_warning "Applications do not exist locally. Creating mock application group structure under '${APP_DIR_IN}/${APP_GROUP}'"
+			mkdir -p "${APP_DIR_IN}/${APP_GROUP}"
 			while read -r APP; do
-				touch "${APP_DIR_IN}/${TARGET_GROUP}/${APP}"
-			done < <(cut -d ',' -f 1 "${REPORTS_DIR}/list__${TARGET_GROUP}__all_apps.csv")
+				touch "${APP_DIR_IN}/${APP_GROUP}/${APP}"
+			done < <(cut -d ',' -f 1 "${REPORTS_DIR}/00__Weave/list__all_apps.csv")
 		else
-			log_console_error "Specified application group ('${TARGET_GROUP}') not found. Please use the import option or organize your applications as described in README.md."
+			log_console_error "Specified application group ('${APP_GROUP}') not found. Please use the import option or organize your applications as described in README.md."
 			ARE_PREREQUISITES_MET=false
 		fi
 	fi
@@ -246,9 +256,9 @@ if [[ "${WINDUP_ACTIVE}" == "true" ]]; then
 		ARE_PREREQUISITES_MET=false
 	fi
 	check_container_engine "${CONTAINER_IMAGE_NAME_WINDUP}" "${DIST_DIR}/oci__windup_${WINDUP_VERSION}.img"
-	if [[ -z "${WINDUP_INCLUDE_PACKAGES_FILE}" && -z "${WINDUP_EXCLUDE_PACKAGES_FILE}" ]]; then
-		log_console_warning "Windup Analysis is active, but no list of packages to include/exclude has been set. It might take a long time to run."
-	fi
+	#if [[ -z "${WINDUP_INCLUDE_PACKAGES_FILE}" && -z "${WINDUP_EXCLUDE_PACKAGES_FILE}" ]]; then
+	#	log_console_warning "Windup Analysis is active, but no list of packages to include/exclude has been set. It might take a long time to run."
+	#fi
 fi
 
 # 04

diff --git a/00__weave_execution_plan.sh b/00__weave_execution_plan.sh
@@ -55,10 +55,31 @@ STEP=$(get_step)
 SEPARATOR=','
 export LOG_FILE=${REPORTS_DIR}/${STEP}__Weave.log
 
+export BASE_DIR="${REPORTS_DIR}/00__Weave"
+
+# All apps
+export LIST_ALL="${BASE_DIR}/list__all_apps.txt"
+export LIST_ALL_INIT="${BASE_DIR}/list__all_init_apps.txt"
+export LANG_MAP_CSV="${BASE_DIR}/list__all_apps.csv"
+
+# Currently the following application type can be analyzed ...
+# -> Java apps decompiled and initially provided as source code apps (zip/folder)
+export LIST_JAVA_SRC="${BASE_DIR}/list__java-src.txt"
+# -> Java binary apps
+export LIST_JAVA_BIN="${BASE_DIR}/list__java-bin.txt"
+# -> Java apps initially provided as source code
+export LIST_JAVA_SRC_INIT="${BASE_DIR}/list__java-src-init.txt"
+# -> JavaScript / Python / C# source code apps
+export LIST_JAVASCRIPT="${BASE_DIR}/list__js.txt"
+export LIST_PYTHON="${BASE_DIR}/list__python.txt"
+export LIST_DOTNET="${BASE_DIR}/list__cs.txt"
+# -> Other apps
+export LIST_OTHER="${BASE_DIR}/list__other.txt"
+
 function identify() {
-	APP_FILE_LIST=${1}
-	APP_SRC_DIR=${2}
-	APP_NAME=$(basename "${APP_SRC_DIR}")
+	local -r APP_FILE_LIST=${1}
+	local -r APP_SRC_DIR=${2}
+	local -r APP_NAME=$(basename "${APP_SRC_DIR}")
 	if (grep -q '.*\.java$' "${APP_FILE_LIST}"); then
 		echo "${APP_SRC_DIR}" >>"${LIST_JAVA_SRC}"
 		echo "${APP_SRC_DIR}" >>"${LIST_JAVA_SRC_INIT}"
@@ -88,71 +109,47 @@ function sort_files() {
 	done
 }
 
-function weave() {
-	GROUP=$(basename "${1}")
-	SRC_DIR="${1}/src"
-	TMP_DIR="${1}/tmp"
-
-	# Remove tmp folder to avoid side effects
-	rm -Rf "${TMP_DIR}"
-
-	# All apps
-	export LIST_ALL=${REPORTS_DIR}/list__${GROUP}__all_apps.txt
-	export LIST_ALL_INIT=${REPORTS_DIR}/list__${GROUP}__all_init_apps.txt
-
-	export LANG_MAP_CSV=${REPORTS_DIR}/list__${GROUP}__all_apps.csv
-
-	# Currently the following application type can be analyzed ...
-	# -> Java apps decompiled and initially provided as source code apps (zip/folder)
-	export LIST_JAVA_SRC=${REPORTS_DIR}/list__${GROUP}__java-src.txt
-	# -> Java binary apps
-	export LIST_JAVA_BIN=${REPORTS_DIR}/list__${GROUP}__java-bin.txt
-	# -> Java apps initially provided as source code
-	export LIST_JAVA_SRC_INIT=${REPORTS_DIR}/list__${GROUP}__java-src-init.txt
-	# -> JavaScript / Python / C# source code apps
-	export LIST_JAVASCRIPT=${REPORTS_DIR}/list__${GROUP}__js.txt
-	export LIST_PYTHON=${REPORTS_DIR}/list__${GROUP}__python.txt
-	export LIST_DOTNET=${REPORTS_DIR}/list__${GROUP}__cs.txt
-	# -> Other apps
-	export LIST_OTHER=${REPORTS_DIR}/list__${GROUP}__other.txt
+function main() {
+	rm -Rf "${BASE_DIR}"
+	mkdir -p "${BASE_DIR}"
 
 	rm -f "${LIST_ALL}" "${LIST_ALL_INIT}" "${LANG_MAP_CSV}" "${LIST_JAVA_SRC}" "${LIST_JAVA_BIN}" "${LIST_JAVA_SRC_INIT}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}"
 	touch "${LIST_ALL}" "${LIST_ALL_INIT}" "${LANG_MAP_CSV}" "${LIST_JAVA_SRC}" "${LIST_JAVA_BIN}" "${LIST_JAVA_SRC_INIT}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}"
 
 	# Add all java binary applications to ${LIST_JAVA_BIN}
-	find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhw]ar' >"${LIST_JAVA_BIN}"
+	find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhw]ar' >"${LIST_JAVA_BIN}"
 
 	# Add decompiled Java apps to ${LIST_JAVA_SRC}
 	while read -r APP; do
 		APP_NAME=$(basename "${APP}")
-		echo "${SRC_DIR}/${APP_NAME}" >>"${LIST_JAVA_SRC}"
+		echo "${APP_GROUP_SRC_DIR}/${APP_NAME}" >>"${LIST_JAVA_SRC}"
 		echo "${APP_NAME}${SEPARATOR}Java" >>"${LANG_MAP_CSV}"
 		echo "${APP}" >>"${LIST_ALL_INIT}"
-	done < <(find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhws]ar')
+	done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.[ejgrhws]ar')
 
 	# Add zipped source code to their category
 	while read -r ARCHIVE; do
 		APP_NAME=$(basename "${ARCHIVE}")
 		APP_FILE_LIST=${ARCHIVE}.lst
 		unzip -Z1 "${ARCHIVE}" >"${APP_FILE_LIST}"
-		identify "${APP_FILE_LIST}" "${SRC_DIR}/${APP_NAME%.*}"
+		identify "${APP_FILE_LIST}" "${APP_GROUP_SRC_DIR}/${APP_NAME%.*}"
 		echo "${ARCHIVE}" >>"${LIST_ALL_INIT}"
-	done < <(find "${1}" -maxdepth 1 -mindepth 1 -type f -name '*.zip')
+	done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type f -name '*.zip')
 
 	# Add exploded source code directories to their category
 	while read -r DIR; do
 		DIR_NAME=$(basename "${DIR}")
 		APP_FILE_LIST=${DIR}.list
 		find "${DIR}" -type f >"${APP_FILE_LIST}"
-		identify "${APP_FILE_LIST}" "${SRC_DIR}/${DIR_NAME}"
+		identify "${APP_FILE_LIST}" "${APP_GROUP_SRC_DIR}/${DIR_NAME}"
 		echo "${DIR}" >>"${LIST_ALL_INIT}"
-	done < <(find "${1}" -maxdepth 1 -mindepth 1 -type d -not -name 'src')
+	done < <(find "${APP_GROUP_DIR}" -maxdepth 1 -mindepth 1 -type d -not -name 'src')
 
 	cat "${LIST_JAVA_SRC}" "${LIST_JAVASCRIPT}" "${LIST_PYTHON}" "${LIST_DOTNET}" "${LIST_OTHER}" >"${LIST_ALL}"
 
 	if [[ "${OWASP_ACTIVE}" == "true" ]]; then
 		# List for OWASP DC
-		export LIST_OWASP_DC=${REPORTS_DIR}/list__${GROUP}__owasp_dc.txt
+		export LIST_OWASP_DC=${BASE_DIR}/list__owasp_dc.txt
 		rm -f "${LIST_OWASP_DC}"
 		touch "${LIST_OWASP_DC}"
 		cat "${LIST_ALL}" >"${LIST_OWASP_DC}"
@@ -172,11 +169,7 @@ function weave() {
 	COUNT_OTHER_APPS=$(count_lines "${LIST_OTHER}")
 	COUNT_ALL_APPS=$(count_lines "${LIST_ALL}")
 
-	log_console_info "[${GROUP}] Identified ${COUNT_ALL_APPS} apps: Java (${COUNT_JAVA_APPS} incl. ${COUNT_JAVA_BIN_APPS} binary), C# (${COUNT_DOTNET_APPS}), JavaScript (${COUNT_JS_APPS}), Python (${COUNT_PY_APPS}), Other (${COUNT_OTHER_APPS})"
-}
-
-function main() {
-	for_each_group weave
+	log_console_info "[${APP_GROUP}] Identified ${COUNT_ALL_APPS} apps: Java (${COUNT_JAVA_APPS} incl. ${COUNT_JAVA_BIN_APPS} binary), C# (${COUNT_DOTNET_APPS}), JavaScript (${COUNT_JS_APPS}), Python (${COUNT_PY_APPS}), Other (${COUNT_OTHER_APPS})"
 }
 
 main
diff --git a/01__fernflower_decompile.sh b/01__fernflower_decompile.sh
@@ -41,8 +41,9 @@ STEP=$(get_step)
 VERSION="${FERNFLOWER_VERSION}"
 
 # List of all archives that have been decompiled
-FERNFLOWER_UNPACKED_LIBS_LIST="${REPORTS_DIR}/${STEP}__Fernflower__unpacked_libs.txt"
-FERNFLOWER_ALL_LIBS_LIST="${REPORTS_DIR}/${STEP}__Fernflower__all_libs.txt"
+BASE_DIR="${REPORTS_DIR}/${STEP}__Fernflower"
+FERNFLOWER_UNPACKED_LIBS_LIST="${BASE_DIR}/list__unpacked_libs.txt"
+FERNFLOWER_ALL_LIBS_LIST="${BASE_DIR}/list__all_libs.txt"
 MVNREPOSITORY_BASE_URL="https://mvnrepository.com"
 
 export LOG_FILE=${REPORTS_DIR}/${STEP}__Fernflower.log
@@ -79,26 +80,19 @@ function unpack() {
 
 # Repack applications to a single huge application without sub-WAR/JAR file
 function unpack_and_decompile() {
-	APP_DIR_INCOMING="${1}"
-	GROUP=$(basename "${APP_DIR_INCOMING}")
-	APP_DIR_SRC="${APP_DIR_INCOMING}/src"
-	APP_DIR_TMP="${APP_DIR_INCOMING}/tmp"
-
-	rm -Rf "${APP_DIR_SRC}" "${APP_DIR_TMP}"
-	mkdir -p "${APP_DIR_SRC}" "${REPORTS_DIR}"
-
-	log_analysis_message "group '${GROUP}'"
+	rm -Rf "${APP_GROUP_SRC_DIR}" "${APP_GROUP_TMP_DIR}"
+	mkdir -p "${APP_GROUP_SRC_DIR}" "${REPORTS_DIR}"
 
 	while read -r APP; do
 
-		mkdir -p "${APP_DIR_TMP}"
+		mkdir -p "${APP_GROUP_TMP_DIR}"
 
 		log_console_step "Preparing '${APP}' ... "
-		cp "${APP}" "${APP_DIR_TMP}"
+		cp "${APP}" "${APP_GROUP_TMP_DIR}"
 
 		APP_NAME=$(basename "${APP}")
 
-		while [ "$(find "${APP_DIR_TMP}" -type f -iname '*.war' \
+		while [ "$(find "${APP_GROUP_TMP_DIR}" -type f -iname '*.war' \
 			-o -type f -iname '*.ear' \
 			-o -type f -iname '*.jar' \
 			-o -type f -iname '*.rar' \
@@ -108,8 +102,8 @@ function unpack_and_decompile() {
 
 			while read -r ARCHIVE; do
 				PARENT_DIR_NAME=$(basename "$(dirname "${ARCHIVE}")")
-				TMP_DIR_NAME=$(basename "${APP_DIR_TMP}")
-				ARCHIVE_SHORT_NAME="${ARCHIVE:${#APP_DIR_TMP}+1}"
+				TMP_DIR_NAME=$(basename "${APP_GROUP_TMP_DIR}")
+				ARCHIVE_SHORT_NAME="${ARCHIVE:${#APP_GROUP_TMP_DIR}+1}"
 				SHA1_LONG=$(sha1sum "${ARCHIVE}")
 				SHA1=$(echo "${SHA1_LONG}" | cut -d' ' -f 1)
 
@@ -237,7 +231,7 @@ function unpack_and_decompile() {
 						mv "${ARCHIVE}" "${ARCHIVE}.ignored_maven"
 					fi
 				fi
-			done < <(find "${APP_DIR_TMP}" -type f -iname '*.war' \
+			done < <(find "${APP_GROUP_TMP_DIR}" -type f -iname '*.war' \
 				-o -type f -iname '*.ear' \
 				-o -type f -iname '*.jar' \
 				-o -type f -iname '*.rar' \
@@ -246,11 +240,11 @@ function unpack_and_decompile() {
 				-o -type f -iname '*.sar')
 		done
 
-		APP_SRC="${APP_DIR_SRC}/${APP_NAME}"
+		APP_SRC="${APP_GROUP_SRC_DIR}/${APP_NAME}"
 		rm -Rf "${APP_SRC}"
 		mkdir -p "${APP_SRC}"
 
-		APP_TMP=${APP_DIR_TMP}/${APP_NAME}
+		APP_TMP=${APP_GROUP_TMP_DIR}/${APP_NAME}
 		APP_TMP_NAME="${APP_TMP%.*}_${APP_TMP##*.}"
 
 		if [[ "${PRE_ANALYSIS_ACTIVE}" == "false" ]]; then
@@ -268,12 +262,12 @@ function unpack_and_decompile() {
 		else
 			log_console_step "Identified '${APP_TMP_NAME}' with a total of $(find "${APP_TMP_NAME}" -name '*.class' | wc -l | tr -d ' ') classes"
 		fi
-		# Note: Here we do not delete ${APP_DIR_TMP} as it creates some issues with the containerized fernflower execution. 
+		# Note: Here we do not delete ${APP_GROUP_TMP_DIR} as it creates some issues with the containerized fernflower execution.
 		rm -Rf "${APP_TMP}" "${APP_TMP_NAME}"
 
-	done <"${REPORTS_DIR}/list__${GROUP}__java-bin.txt"
+	done <"${REPORTS_DIR}/00__Weave/list__java-bin.txt"
 
-	rm -Rf "${APP_DIR_TMP}"
+	rm -Rf "${APP_GROUP_TMP_DIR}"
 
 }
 
@@ -287,24 +281,24 @@ function check_status() {
 
 function main() {
 
-	if [[ "${DEBUG}" == "true" ]]; then
-		set -x
-		exec 6>&1
-		FERNFLOWER_LOG_LEVEL="INFO"
-	else
-		exec 6>/dev/null
-	fi
+	log_tool_info "Fernflower v${VERSION}"
 
 	if [[ -n $(${CONTAINER_ENGINE} images -q "${CONTAINER_IMAGE_NAME_FERNFLOWER}") ]]; then
 
+		check_debug_mode
+		if [[ "${DEBUG}" == "true" ]]; then
+			FERNFLOWER_LOG_LEVEL="INFO"
+		fi
+
 		set +e
-		log_tool_info "Fernflower v${VERSION}"
 
 		[[ "${USE_FINDJAR}" == "true" ]] && { check_status "findjar.com" "${FINDJAR_BASE_URL}"; }
 		check_status "Maven Search" "${MAVEN_SEARCH_BASE_URL}"
 		check_status "MVN Repository" "${MVNREPOSITORY_BASE_URL}"
 
-		for_each_group unpack_and_decompile
+		mkdir -p "${BASE_DIR}"
+
+		unpack_and_decompile
 
 		if [[ "${PRE_ANALYSIS_ACTIVE}" == "true" ]]; then