diff --git a/kube/services/jobs/compare-gen3sdk-dep-versions-job.yaml b/kube/services/jobs/compare-gen3sdk-dep-versions-job.yaml new file mode 100644 index 000000000..bac31a694 --- /dev/null +++ b/kube/services/jobs/compare-gen3sdk-dep-versions-job.yaml @@ -0,0 +1,156 @@ +# +# run with: +# gen3 job run compare-gen3sdk-dep-versions [VERBOSE true] [SHOW_VERSIONS all|mismatched|mismatched_major] +# +# VERBOSE true +# Log standard output from 'pip install' and 'poetry show' commands. +# Default is to log only the different/all dependency versions in csv format. +# +# SHOW VERSIONS +# all - all dependency versions. +# mismatched - dependencies with different versions. +# mismatched_major - dependencies with different major versions. + +apiVersion: batch/v1 +kind: Job +metadata: + name: compare-gen3sdk-dep-versions +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: gen3job + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - weight: 99 + preference: + matchExpressions: + - key: eks.amazonaws.com/capacityType + operator: In + values: + - ONDEMAND + serviceAccountName: useryaml-job + volumes: + - name: shared-data + emptyDir: {} + containers: + - name: awshelper + GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-| + imagePullPolicy: Always + ports: + - containerPort: 80 + env: + - name: VERBOSE + GEN3_VERBOSE|-value: ""-| + - name: SHOW_VERSIONS + GEN3_SHOW_VERSIONS|-value: ""-| + volumeMounts: + - name: shared-data + mountPath: /mnt/shared + resources: + limits: + cpu: 1 + memory: 1Gi + + command: ["/bin/bash" ] + args: + - "-c" + - | + if [ "$VERBOSE" = true ]; then + pip --version + pip install gen3 + else + pip install gen3 > /dev/null 2>&1 + fi + pip freeze | sed 's/==/\t/' | sort > pip_freeze.tsv + if [ "$VERBOSE" = true ]; then + cat pip_freeze.tsv + fi + + # get the gen3sdk version stored in pypi + GEN3_PYPI_VERSION=`grep 'gen3\s' pip_freeze.tsv | awk '{print$2}'` + if [ "$VERBOSE" = true ]; then + echo "Gen3sdk version in PyPI: ${GEN3_PYPI_VERSION}" + fi + if [ -z "${GEN3_PYPI_VERSION}" ]; then + echo -e "ERROR: could not get gen3sdk version from pip freeze output" + exit 0 + fi + + # poetry - use poetry lock file for gen3sdk repo tag that matches pypi version + GITHUB_REPO="raw.githubusercontent.com/uc-cdis/gen3sdk-python/${GEN3_PYPI_VERSION}" + if [ "$VERBOSE" = true ]; then + echo "Using github repo: ${GITHUB_REPO}" + fi + curl -OJLs https://${GITHUB_REPO}/poetry.lock + curl -OJLs https://${GITHUB_REPO}/pyproject.toml + if [[ ! -f poetry.lock || ! -f pyproject.toml ]]; then + echo -e "ERROR: could not get poetry.lock or pyproject.toml file from ${GITHUB_REPO}" + exit 0 + fi + # install the version of poetry that matches the lock file + VERSION_STRING=`grep "generated by Poetry" poetry.lock` + rx='Poetry ([0-9]+\.[0-9]+\.[0-9]+)' + if [[ "$VERSION_STRING" =~ $rx ]]; then + POETRY_VERSION="${BASH_REMATCH[1]}" + else + echo "ERROR: Could not get poetry version from lock file" + exit 0 + fi + if [ "$VERBOSE" = true ]; then + echo "Installing poetry version $POETRY_VERSION" + pip install poetry=="$POETRY_VERSION" + PATH=$PATH:~/.local/bin + poetry --version + else + pip install poetry=="$POETRY_VERSION" > /dev/null 2>&1 + PATH=$PATH:~/.local/bin + fi + poetry show 2>/dev/null | sed 's/(\!)//' | awk '{print $1"\t"$2}' | sort > poetry_show.tsv + if [ "$VERBOSE" = true ]; then + echo "Poetry-show file" + cat poetry_show.tsv + echo "" + fi + + # compare + join -j 1 -o 1.1,1.2,2.2 poetry_show.tsv pip_freeze.tsv | awk '{print $1","$2","$3}' > gen3_deps_versions.csv + cat gen3_deps_versions.csv | awk -F ',' '$2 != $3' > mismatched_versions.csv + + if [ "${SHOW_VERSIONS}" = "mismatched_major" ]; then + while read v; do + rx='.+,([0-9]+)\.[0-9]+\.?.*,([0-9]+)\.[0-9]+\.?.*' + if [[ "$v" =~ $rx ]]; then + POE_VER=${BASH_REMATCH[1]} + PIP_VER=${BASH_REMATCH[2]} + DIFF=$((${PIP_VER} - ${POE_VER})) + # get differences in major versions, skip calvers + if [[ ${DIFF#-} -ge 1 && ${POE_VER} -le 2020 && ${PIP_VER} -le 2020 ]]; then + touch /mnt/shared/found_major_version_diffs + echo "${v}" >> mismatched_major_versions.csv + fi + fi + done < mismatched_versions.csv + fi + + echo "package,poetry-version,pip-version" + echo "gen3,${GEN3_PYPI_VERSION},${GEN3_PYPI_VERSION}" + if [ "${SHOW_VERSIONS}" = "all" ]; then + cat gen3_libs_versions.csv + elif [ "${SHOW_VERSIONS}" = "mismatched_major" ]; then + cat mismatched_major_versions.csv + else + cat mismatched_versions.csv + fi + + restartPolicy: Never \ No newline at end of file