diff --git a/.github/workflows/builder-unit-test.yml b/.github/workflows/builder-unit-test.yml index 8f2c784..dc15602 100644 --- a/.github/workflows/builder-unit-test.yml +++ b/.github/workflows/builder-unit-test.yml @@ -68,13 +68,13 @@ jobs: run: | pip install -e . pytest tests/ - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - directory: ./ - flags: unittests - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: true - path_to_write_report: ./codecov_report.gz + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v1 + #with: + #file: ./coverage.xml + #directory: ./ + #flags: unittests + #env_vars: OS,PYTHON + #name: codecov-umbrella + #fail_ci_if_error: true + #path_to_write_report: ./codecov_report.gz diff --git a/open_ce/container_build.py b/open_ce/container_build.py index f257560..e7ecf61 100644 --- a/open_ce/container_build.py +++ b/open_ce/container_build.py @@ -26,9 +26,9 @@ from open_ce.inputs import Argument OPEN_CE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".")) -BUILD_IMAGE_NAME = "builder" +BUILD_IMAGE_NAME = "builder/ubi8" BUILD_IMAGE_PATH = os.path.join(OPEN_CE_PATH, "images", BUILD_IMAGE_NAME) -BUILD_CUDA_IMAGE_NAME = "builder-cuda" +BUILD_CUDA_IMAGE_NAME = "builder-cuda/ubi8" BUILD_CUDA_IMAGE_PATH = os.path.join(OPEN_CE_PATH, "images", BUILD_CUDA_IMAGE_NAME) LOCAL_FILES_PATH = os.path.join(os.path.join(os.getcwd(), "local_files")) diff --git a/open_ce/images/builder-cuda/Dockerfile.cuda-11.8 b/open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-11.8 similarity index 100% rename from open_ce/images/builder-cuda/Dockerfile.cuda-11.8 rename to open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-11.8 diff --git a/open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-12.2 b/open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-12.2 new file mode 100644 index 0000000..a818a88 --- /dev/null +++ b/open_ce/images/builder-cuda/ubi8/Dockerfile.cuda-12.2 @@ -0,0 +1,59 @@ +FROM docker.io/nvidia/cuda:12.2.2-devel-ubi8 + +ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} +ENV PATH=$CONDA_HOME/bin:$PATH +ENV CUDA_HOME=/usr/local/cuda + +ENV OPEN_CE_CONDA_BUILD=3.22.0 +ENV OPEN_CE_CONDA=4.14 +ENV OPENSSL_VER=3.* + +ENV CICD_GROUP=cicd +ARG GROUP_ID=1500 +ENV BUILD_USER=builder +ARG BUILD_ID=1084 + +ARG LIMIT_BUILD_RESOURCES=0 +ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} + +# Remove preinclude system compilers +RUN rpm -e --nodeps --verbose gcc gcc-c++ + +RUN export ARCH="$(uname -m)" && \ + yum repolist && yum install -y rsync openssh-clients diffutils procps git-lfs file psmisc openssl-devel && \ + # Create CICD Group + groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ + # Adduser Builder + useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ + --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ + curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ + chmod +x /tmp/anaconda.sh && \ + /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ + rm -f /tmp/anaconda.sh && \ + $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ + $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ + $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ + $CONDA_HOME/bin/conda config --system --set always_yes true && \ + $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ + $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ + $CONDA_HOME/bin/conda --version && \ + $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ + mkdir -p $CONDA_HOME/conda-bld && \ + mkdir -p $HOME/.cache && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 + + +USER ${BUILD_USER} +RUN export PATH="${PATH}" && \ + echo "PATH="${PATH}"" >> ${HOME}/.profile && \ + mkdir -p $HOME/.cache && \ + echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + echo "conda activate base" >> ${HOME}/.bashrc && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 diff --git a/open_ce/images/builder-cuda/Dockerfile.cuda-12.2 b/open_ce/images/builder-cuda/ubi9/Dockerfile.cuda-12.2 similarity index 100% rename from open_ce/images/builder-cuda/Dockerfile.cuda-12.2 rename to open_ce/images/builder-cuda/ubi9/Dockerfile.cuda-12.2 diff --git a/open_ce/images/builder/ubi8/Dockerfile b/open_ce/images/builder/ubi8/Dockerfile new file mode 100644 index 0000000..a0fdd0d --- /dev/null +++ b/open_ce/images/builder/ubi8/Dockerfile @@ -0,0 +1,55 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} +ENV PATH=$CONDA_HOME/bin:$PATH + +ENV OPEN_CE_CONDA_BUILD=3.22.0 +ENV OPEN_CE_CONDA=4.14 +ENV OPENSSL_VER=3.* + +ENV CICD_GROUP=cicd +ARG GROUP_ID=1500 +ENV BUILD_USER=builder +ARG BUILD_ID=1084 + +ARG LIMIT_BUILD_RESOURCES=0 +ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} + +RUN export ARCH="$(uname -m)" && \ + yum repolist && yum install -y rsync openssh-clients diffutils procps binutils git-lfs glibc-devel file psmisc openssl-devel && \ + # Create CICD Group + groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ + # Adduser Builder + useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ + --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ + curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ + chmod +x /tmp/anaconda.sh && \ + /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ + rm -f /tmp/anaconda.sh && \ + $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ + $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ + $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ + $CONDA_HOME/bin/conda config --system --set always_yes true && \ + $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ + $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ + $CONDA_HOME/bin/conda --version && \ + $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ + mkdir -p $CONDA_HOME/conda-bld && \ + mkdir -p $HOME/.cache && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 + +USER ${BUILD_USER} +RUN export PATH="${PATH}" && \ + echo "PATH="${PATH}"" >> ${HOME}/.profile && \ + mkdir -p $HOME/.cache && \ + echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + echo "conda activate base" >> ${HOME}/.bashrc && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 + diff --git a/open_ce/images/builder/ubi8/Dockerfile-p10 b/open_ce/images/builder/ubi8/Dockerfile-p10 new file mode 100644 index 0000000..50c12c3 --- /dev/null +++ b/open_ce/images/builder/ubi8/Dockerfile-p10 @@ -0,0 +1,66 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ENV CONDA_HOME=${CONDA_HOME:-/opt/conda} +ENV PATH=$CONDA_HOME/bin:$PATH + +ENV OPEN_CE_CONDA_BUILD=3.22.0 +ENV OPEN_CE_CONDA=4.14 +ENV OPENSSL_VER=3.* + +ENV CICD_GROUP=cicd +ARG GROUP_ID=1500 +ENV BUILD_USER=builder +ARG BUILD_ID=1084 + +ARG LIMIT_BUILD_RESOURCES=0 +ENV LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES} + +RUN export ARCH="$(uname -m)" && \ + yum install -y yum-utils rsync openssh-clients diffutils procps git-lfs gcc-toolset-12 glibc-devel file psmisc libX11-devel openssl openssl-libs openssl-devel && \ + yum install -y libtirpc-devel && \ + # Create CICD Group + groupadd --non-unique --gid ${GROUP_ID} ${CICD_GROUP} && \ + # Adduser Builder + useradd -b /home --non-unique --create-home --gid ${GROUP_ID} --groups wheel \ + --uid ${BUILD_ID} --comment "User for Building" ${BUILD_USER} && \ + curl -o /tmp/anaconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py310_22.11.1-1-Linux-${ARCH}.sh && \ + chmod +x /tmp/anaconda.sh && \ + /bin/bash /tmp/anaconda.sh -f -b -p /opt/conda && \ + rm -f /tmp/anaconda.sh && \ + $CONDA_HOME/bin/conda install -y conda-build=${OPEN_CE_CONDA_BUILD} conda=${OPEN_CE_CONDA} networkx git junit-xml patch && \ + $CONDA_HOME/bin/conda config --system --add envs_dirs $CONDA_HOME/envs && \ + $CONDA_HOME/bin/conda config --system --add pkgs_dirs $CONDA_HOME/pkgs && \ + $CONDA_HOME/bin/conda config --system --set always_yes true && \ + $CONDA_HOME/bin/conda config --system --set auto_update_conda false && \ + $CONDA_HOME/bin/conda config --system --set notify_outdated_conda false && \ + $CONDA_HOME/bin/conda config --system --add create_default_packages openssl=${OPENSSL_VER} && \ + $CONDA_HOME/bin/conda --version && \ + $CONDA_HOME/bin/conda install openssl=${OPENSSL_VER} && \ + mkdir -p $CONDA_HOME/conda-bld && \ + mkdir -p $HOME/.cache && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ + export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ + echo "PATH="${PATH}"" >> ${HOME}/.profile && \ + echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ + echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ + chown -R ${BUILD_USER}:${CICD_GROUP} ${CONDA_HOME} && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 + + +USER ${BUILD_USER} +RUN export PATH="/opt/rh/gcc-toolset-12/root/usr/bin:${PATH}" && \ + export GCC_HOME="/opt/rh/gcc-toolset-12/root/usr" && \ + echo "PATH="${PATH}"" >> ${HOME}/.profile && \ + echo "PATH="${PATH}"" >> ${HOME}/.bashrc && \ + echo "GCC_HOME="${GCC_HOME}"" >> ${HOME}/.bashrc && \ + mkdir -p $HOME/.cache && \ + echo ". $CONDA_HOME/etc/profile.d/conda.sh" >> ${HOME}/.bashrc && \ + echo "export PYTHONPATH=${PYTHONPATH}:$HOME/open_ce" >> ${HOME}/.bashrc && \ + echo "export LIMIT_BUILD_RESOURCES=${LIMIT_BUILD_RESOURCES}" >> ${HOME}/.bashrc && \ + echo "conda activate base" >> ${HOME}/.bashrc && \ + git config --global http.version HTTP/1.1 && \ + git config --global http.postBuffer 157286400 + diff --git a/open_ce/images/builder/Dockerfile-s390x b/open_ce/images/builder/ubi8/Dockerfile-s390x similarity index 100% rename from open_ce/images/builder/Dockerfile-s390x rename to open_ce/images/builder/ubi8/Dockerfile-s390x diff --git a/open_ce/images/builder/Dockerfile b/open_ce/images/builder/ubi9/Dockerfile similarity index 100% rename from open_ce/images/builder/Dockerfile rename to open_ce/images/builder/ubi9/Dockerfile diff --git a/open_ce/images/builder/Dockerfile-p10 b/open_ce/images/builder/ubi9/Dockerfile-p10 similarity index 100% rename from open_ce/images/builder/Dockerfile-p10 rename to open_ce/images/builder/ubi9/Dockerfile-p10