SynapseAi 1.19.0 release

* Update dockerfiles with 1.19.0 content
HabanaAI · Dec 19, 2024 · 5ce1081 · 5ce1081
1 parent 5ec0d0c
commit 5ce1081
Show file tree

Hide file tree

Showing 26 changed files with 155 additions and 384 deletions.
diff --git a/dockerfiles/base/Dockerfile.amzn2 b/dockerfiles/base/Dockerfile.amzn2
diff --git a/dockerfiles/base/Dockerfile.rhel8.6 b/dockerfiles/base/Dockerfile.rhel8.6
@@ -37,7 +37,8 @@ RUN dnf install -y \
     unzip \
     llvm \
     lsof \
-    python38-devel \
+    python3.11-devel \
+    python3.11-pip \
     bzip2 \
     bzip2-devel \
     openssh-clients \
@@ -93,6 +94,11 @@ RUN echo "[powertools]" > /etc/yum.repos.d/powertools.repo && \
     echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/powertools.repo && \
     echo "gpgcheck=1" >> /etc/yum.repos.d/powertools.repo
 
+ENV PYTHON_VERSION=3.11
+RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
+    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
+    alternatives --set python3 /usr/bin/python3.11
+
 RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el8 \
         habanalabs-thunk-"$VERSION"-"$REVISION".el8 \
         habanalabs-firmware-tools-"$VERSION"-"$REVISION".el8 \
@@ -127,14 +133,9 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-ENV PYTHON_VERSION=3.8
-RUN python3.8 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
-
-RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
-    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
-    alternatives --set python3 /usr/bin/python3.8
+RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
-RUN python3.8 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
+RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 
 # SSH configuration necessary to support mpi-operator v2
 RUN mkdir -p /var/run/sshd && \
@@ -147,4 +148,4 @@ RUN mkdir -p /var/run/sshd && \
 ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so
 ENV HABANA_LOGS=/var/log/habana_logs/
 ENV HABANA_SCAL_BIN_PATH=/opt/habanalabs/engines_fw
-ENV HABANA_PLUGINS_LIB_PATH=/opt/habanalabs/habana_plugins
+ENV HABANA_PLUGINS_LIB_PATH=/opt/habanalabs/habana_plugins
diff --git a/dockerfiles/base/Dockerfile.rhel9.2 b/dockerfiles/base/Dockerfile.rhel9.2
@@ -68,6 +68,7 @@ RUN dnf install -y \
 ENV PYTHON_VERSION=3.10
 COPY install-python310.sh .
 RUN ./install-python310.sh rhel9.2 && rm install-python310.sh
+RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && ldconfig
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
 COPY install_efa.sh .
@@ -128,11 +129,11 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.10 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
+RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
-RUN python3.10 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
+RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 
 # SSH configuration necessary to support mpi-operator v2
 RUN mkdir -p /var/run/sshd && \

diff --git a/dockerfiles/base/Dockerfile.rhel9.4 b/dockerfiles/base/Dockerfile.rhel9.4
@@ -31,10 +31,15 @@ RUN echo "[CRB]" > /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
     echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
     echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo
 
-RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
-    dnf clean all && rm -rf /var/cache/yum
-
 RUN dnf install -y \
+        python3-dnf-plugin-versionlock && \
+    dnf versionlock add redhat-release* && \
+    dnf clean all
+
+RUN dnf update -y && dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    dnf clean all
+
+RUN dnf update -y && dnf install -y \
     clang \
     cmake3 \
     cpp \
@@ -51,8 +56,8 @@ RUN dnf install -y \
     lsof \
     python3-devel \
     openssh-clients \
-    openssl-1:3.0.7-28.el9_4 \
-    openssl-devel-1:3.0.7-28.el9_4 \
+    openssl \
+    openssl-devel \
     libjpeg-devel \
     openssh-server \
     lsb_release \
@@ -68,12 +73,9 @@ RUN dnf install -y \
     python3.11-pip \
     python3.11-devel \
     python3.11-rpm \
-    ffmpeg-free \
-    python3-dnf-plugin-versionlock && \
-    # update pkgs (except OS version) for resolving potentials CVEs
-    dnf versionlock add redhat-release* openssl* libcurl-minimal curl-minimal ima-evm-utils python3-rpm rpm* && \
-    dnf update -y && \
-    dnf clean all && rm -rf /var/cache/yum && \
+    ffmpeg-free && \
+    dnf versionlock add python3-rpm rpm* && \
+    dnf clean all && \
     rm -f /etc/ssh/ssh_host_*_key*
 
 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
@@ -111,8 +113,9 @@ RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
         habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
         habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
         habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
-    rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
-    dnf clean all && rm -rf /var/cache/yum
+    chmod +t /var/log/habana_logs && \
+    rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && \
+    dnf clean all
 
 RUN rpm -V habanalabs-rdma-core && rpm -V habanalabs-thunk && rpm -V habanalabs-firmware-tools && rpm -V habanalabs-graph
 
@@ -141,11 +144,11 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.11 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
+RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
-RUN python3.11 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
+RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 
 # SSH configuration necessary to support mpi-operator v2
 RUN mkdir -p /var/run/sshd && \

diff --git a/dockerfiles/base/Dockerfile.suse15.5 b/dockerfiles/base/Dockerfile.suse15.5
@@ -44,7 +44,8 @@ RUN zypper install -y --allow-downgrade \
     Mesa-libGL1 \
     openssh-clients \
     openssh-server \
-    openssl openssl-devel \
+    openssl \
+    openssl-devel \
     python311 \
     python311-devel \
     python311-pip \
@@ -53,11 +54,14 @@ RUN zypper install -y --allow-downgrade \
     zlib-devel && \
     rm -f /etc/ssh/ssh_host_*_key*
 
+ENV PIP_NO_CACHE_DIR=on
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+
 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
     alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
-    alternatives --set python3 /usr/bin/python3.11 && \
-    alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.11 1 && \
-    alternatives --set pip3 /usr/bin/pip3.11
+    alternatives --set python3 /usr/bin/python3.11
+
+RUN python3 -m pip install setuptools==75.1.0 wheel==0.44.0
 
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
@@ -82,10 +86,8 @@ RUN zypper --gpg-auto-import-keys install -y habanalabs-rdma-core-"$VERSION"-"$R
         habanalabs-thunk-"$VERSION"-"$REVISION" \
         habanalabs-firmware-tools-"$VERSION"-"$REVISION" \
         habanalabs-graph-"$VERSION"-"$REVISION" && \
-    rm -f /etc/yum.repos.d/habanalabs.repo
+    rm -f /etc/zypp/repos.d/habanalabs.repo
 
-ENV PIP_NO_CACHE_DIR=on
-ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 ENV RDMA_CORE_ROOT=/opt/habanalabs/rdma-core/src
 ENV RDMA_CORE_LIB=${RDMA_CORE_ROOT}/build/lib
 
@@ -102,10 +104,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.11 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
-
-
-RUN python3.11 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
+RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 
 # SSH configuration necessary to support mpi-operator v2
 RUN mkdir -p /var/run/sshd && \

diff --git a/dockerfiles/base/Dockerfile.tencentos3.1 b/dockerfiles/base/Dockerfile.tencentos3.1
@@ -47,13 +47,14 @@ RUN dnf install -y \
 
 COPY install-python310.sh .
 RUN ./install-python310.sh tencentos3.1 && rm install-python310.sh
+RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && ldconfig
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
 COPY install_efa.sh .
 COPY tencentos_efa_patch.txt /tmp/tencentos_efa_patch.txt
 RUN ./install_efa.sh && rm -f install_efa.sh /tmp/tencentos_efa_patch.txt && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
-ENV LIBFABRIC_VERSION="1.20.0"
+ENV LIBFABRIC_VERSION="1.22.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
 ENV MPI_ROOT=/opt/amazon/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH

diff --git a/dockerfiles/base/Dockerfile.ubuntu22.04 b/dockerfiles/base/Dockerfile.ubuntu22.04
@@ -32,6 +32,7 @@ RUN apt-get update && \
     libgl1 \
     libgoogle-glog0v5 \
     libjemalloc2 \
+    libjpeg-dev \
     libpq-dev \
     lsof \
     make \
@@ -60,7 +61,7 @@ RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
-ENV LIBFABRIC_VERSION="1.20.0"
+ENV LIBFABRIC_VERSION="1.22.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
 ENV MPI_ROOT=/opt/amazon/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH

diff --git a/dockerfiles/base/Dockerfile.ubuntu24.04 b/dockerfiles/base/Dockerfile.ubuntu24.04
@@ -32,6 +32,7 @@ RUN apt-get update && \
     libgl1 \
     libgoogle-glog0v6t64 \
     libjemalloc2 \
+    libjpeg-dev \
     libpq-dev \
     lsof \
     make \
@@ -60,7 +61,7 @@ RUN python3 -m pip install pip==24.0 setuptools==75.1.0 wheel==0.42.0 --break-sy
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
-ENV LIBFABRIC_VERSION="1.20.0"
+ENV LIBFABRIC_VERSION="1.22.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
 ENV MPI_ROOT=/opt/amazon/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH