Feat/add gen3 dep version checker

kube/services/jobs/compare-gen3sdk-dep-versions-job.yaml

@@ -0,0 +1,156 @@
+#
+# run with:
+# gen3 job run compare-gen3sdk-dep-versions [VERBOSE true] [SHOW_VERSIONS all|mismatched|mismatched_major]
+#
+# VERBOSE true
+#   Log standard output from 'pip install' and 'poetry show' commands.
+#   Default is to log only the different/all dependency versions in csv format.
+#
+# SHOW VERSIONS
+#  all - all dependency versions.
+#  mismatched - dependencies with different versions.
+#  mismatched_major - dependencies with different major versions.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: compare-gen3sdk-dep-versions
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: gen3job
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      serviceAccountName: useryaml-job
+      volumes:
+        - name: shared-data
+          emptyDir: {}
+      containers:
+      - name: awshelper
+        GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-|
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 80
+        env:
+        - name: VERBOSE
+          GEN3_VERBOSE|-value: ""-|
+        - name: SHOW_VERSIONS
+          GEN3_SHOW_VERSIONS|-value: ""-|
+        volumeMounts:
+        - name: shared-data
+          mountPath: /mnt/shared
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+
+        command: ["/bin/bash" ]
+        args:
+          - "-c"
+          - |
+            if [ "$VERBOSE" = true ]; then
+                pip --version
+                pip install gen3
+            else
+                pip install gen3 > /dev/null 2>&1
+            fi
+            pip freeze | sed 's/==/\t/' | sort > pip_freeze.tsv
+            if [ "$VERBOSE" = true ]; then
+                cat pip_freeze.tsv
+            fi
+
+            # get the gen3sdk version stored in pypi
+            GEN3_PYPI_VERSION=`grep 'gen3\s' pip_freeze.tsv | awk '{print$2}'`
+            if [ "$VERBOSE" = true ]; then
+                echo "Gen3sdk version in PyPI: ${GEN3_PYPI_VERSION}"
+            fi
+            if [ -z "${GEN3_PYPI_VERSION}" ]; then
+                echo -e "ERROR: could not get gen3sdk version from pip freeze output"
+                exit 0
+            fi
+
+            # poetry - use poetry lock file for gen3sdk repo tag that matches pypi version
+            GITHUB_REPO="raw.githubusercontent.com/uc-cdis/gen3sdk-python/${GEN3_PYPI_VERSION}"
+            if [ "$VERBOSE" = true ]; then
+                echo "Using github repo: ${GITHUB_REPO}"
+            fi
+            curl -OJLs https://${GITHUB_REPO}/poetry.lock
+            curl -OJLs https://${GITHUB_REPO}/pyproject.toml
+            if [[ ! -f poetry.lock || ! -f pyproject.toml ]]; then
+                echo -e "ERROR: could not get poetry.lock or pyproject.toml file from ${GITHUB_REPO}"
+                exit 0
+            fi
+            # install the version of poetry that matches the lock file
+            VERSION_STRING=`grep "generated by Poetry" poetry.lock`
+            rx='Poetry ([0-9]+\.[0-9]+\.[0-9]+)'
+            if [[ "$VERSION_STRING" =~ $rx ]]; then
+                POETRY_VERSION="${BASH_REMATCH[1]}"
+            else
+                echo "ERROR: Could not get poetry version from lock file"
+                exit 0
+            fi
+            if [ "$VERBOSE" = true ]; then
+                echo "Installing poetry version $POETRY_VERSION"
+                pip install poetry=="$POETRY_VERSION"
+                PATH=$PATH:~/.local/bin
+                poetry --version
+            else
+                pip install poetry=="$POETRY_VERSION" > /dev/null 2>&1
+                PATH=$PATH:~/.local/bin
+            fi
+            poetry show 2>/dev/null | sed 's/(\!)//' | awk '{print $1"\t"$2}' | sort > poetry_show.tsv
+            if [ "$VERBOSE" = true ]; then
+                echo "Poetry-show file"
+                cat poetry_show.tsv
+                echo ""
+            fi            
+
+            # compare
+            join -j 1 -o 1.1,1.2,2.2 poetry_show.tsv pip_freeze.tsv | awk '{print $1","$2","$3}' > gen3_deps_versions.csv
+            cat gen3_deps_versions.csv | awk -F ',' '$2 != $3' > mismatched_versions.csv
+
+            if [ "${SHOW_VERSIONS}" = "mismatched_major" ]; then
+                while read v; do
+                rx='.+,([0-9]+)\.[0-9]+\.?.*,([0-9]+)\.[0-9]+\.?.*'
+                if [[ "$v" =~ $rx ]]; then
+                    POE_VER=${BASH_REMATCH[1]}
+                    PIP_VER=${BASH_REMATCH[2]}
+                    DIFF=$((${PIP_VER} - ${POE_VER}))
+                    # get differences in major versions, skip calvers
+                    if [[ ${DIFF#-} -ge 1 && ${POE_VER} -le 2020 && ${PIP_VER} -le 2020 ]]; then
+                        touch /mnt/shared/found_major_version_diffs
+                        echo "${v}" >> mismatched_major_versions.csv
+                    fi
+                fi
+                done < mismatched_versions.csv
+            fi
+
+            echo "package,poetry-version,pip-version"
+            echo "gen3,${GEN3_PYPI_VERSION},${GEN3_PYPI_VERSION}"
+            if [ "${SHOW_VERSIONS}" = "all" ]; then
+                cat gen3_libs_versions.csv
+            elif [ "${SHOW_VERSIONS}" = "mismatched_major" ]; then
+                cat mismatched_major_versions.csv
+            else
+                cat mismatched_versions.csv
+            fi
+
+      restartPolicy: Never