SynapseAi 1.18.0 release

* Update dockerfiles with 1.18.0 content
HabanaAI · Oct 11, 2024 · 5ec0d0c · 5ec0d0c
1 parent fbba3e5
commit 5ec0d0c
Show file tree

Hide file tree

Showing 41 changed files with 1,770 additions and 296 deletions.
diff --git a/dockerfiles/README.md b/dockerfiles/README.md
@@ -49,10 +49,27 @@ This script can be used as reference to build docker images for Gaudi.
     make build
     ```
 
+    #### Build triton vllm backend (default OS - ubuntu22.04):
+    ```
+    cd triton_vllm_backend
+    make build BUILD_OS=ubuntu22.04
+    ```
+
 3. Build command variables
 
     #### Optional Parameters
     * BUILD_OS - set the OS to build (default ubuntu22.04)
     * BUILD_DIR - the folder where the build be executed from (default dockerbuild in image folder)
     * VERBOSE - set to TRUE to echo the commands (default FALSE)
-    * DOCKER_CACHE - set to TRUE to use cache for building docker image (default FALSE)
+    * DOCKER_CACHE - set to TRUE to use cache for building docker image (default FALSE)
+
+4. Instructions for triton-vllm-back-end server
+
+   * Run the backend container as described in [habana docs](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Triton_Inference.html?highlight=triton%20inference#run-the-backend-container)
+   * Start the triton server
+     ```bash
+     tritonserver --model-repository samples/model_repository
+     ```
+     The current samples/model_repository/vllm_model contains llama27B 1x.We also have sample model files for llama2 7b/70b and qwen2-7b respectively under samples/model_repository/test_models folder. To use them , copy the model.json and config.pbtxt to vllm_model folder structure.
+   * To test with client, please follow the instructions [here](https://github.com/triton-inference-server/vllm_backend?tab=readme-ov-file#sending-your-first-inference)
+
diff --git a/dockerfiles/base/Dockerfile.amzn2 b/dockerfiles/base/Dockerfile.amzn2
@@ -24,8 +24,10 @@ RUN amazon-linux-extras enable python3.8 && \
     wget \
     lsof \
     tar \
-    mesa-libGL && \
-    yum clean all && rm -rf /var/cache/yum
+    mesa-libGL \
+    sox-devel && \
+    yum clean all && rm -rf /var/cache/yum && \
+    rm -f /etc/ssh/ssh_host_*_key*
 
 # Install jemalloc-3.6.0-1.el7.x86_64 package with required /lib64/libjemalloc.so.1 lib need for topologies
 RUN yum install -y https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm && \
@@ -39,7 +41,7 @@ RUN yum install -y sudo system-lsb-core cmake
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
-ENV LIBFABRIC_VERSION="1.20.0"
+ENV LIBFABRIC_VERSION="1.22.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
 ENV MPI_ROOT=/opt/amazon/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
@@ -74,7 +76,6 @@ RUN sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config &&
     sed -i 's/[ #]\(.*ForwardAgent \).*/ \1yes/g' /etc/ssh/ssh_config && \
     echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
     sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
-    ssh-keygen -A && \
     mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc
 
 # There is no need to store pip installation files inside docker image
@@ -94,7 +95,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
+RUN python3 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN python3 -m pip install habana_media_loader=="${VERSION}"."${REVISION}"
 

diff --git a/dockerfiles/base/Dockerfile.rhel8.6 b/dockerfiles/base/Dockerfile.rhel8.6
@@ -38,6 +38,8 @@ RUN dnf install -y \
     llvm \
     lsof \
     python38-devel \
+    bzip2 \
+    bzip2-devel \
     openssh-clients \
     libjpeg-devel \
     openssh-server \
@@ -50,7 +52,8 @@ RUN dnf install -y \
     # update pkgs (except OS version) for resolving potentials CVEs
     dnf versionlock add redhat-release* && \
     dnf update -y && \
-    dnf clean all && rm -rf /var/cache/yum
+    dnf clean all && rm -rf /var/cache/yum && \
+    rm -f /etc/ssh/ssh_host_*_key*
 
 # CVE-2023-47038 RHSA-2024:3128
 RUN dnf module reset perl -y && \
@@ -67,15 +70,17 @@ RUN echo "[appstream]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
+ENV OPENMPI_VERSION=4.1.6
 ENV LIBFABRIC_VERSION="1.20.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
-ENV MPI_ROOT=/opt/amazon/openmpi
+ENV MPI_ROOT=/opt/habanalabs/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
 ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
 ENV OPAL_PREFIX=${MPI_ROOT}
 ENV MPICC=${MPI_ROOT}/bin/mpicc
 ENV RDMAV_FORK_SAFE=1
-ENV FI_EFA_USE_DEVICE_RDMA=1
+ENV FI_EFA_USE_DEVICE_RDMA=0
+ENV OMPI_MCA_btl=^openib
 
 RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
     echo "name=Habana RH8 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
@@ -109,6 +114,12 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}
 
+RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
+    tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
+    cd /tmp/openmpi-${OPENMPI_VERSION} && \
+    ./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
+    make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}
+
 RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
     unzip /tmp/main.zip -d /tmp && \
     cd /tmp/hccl_ofi_wrapper-main && \
@@ -117,7 +128,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
 ENV PYTHON_VERSION=3.8
-RUN python3.8 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
+RUN python3.8 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \
     alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 && \
@@ -131,7 +142,6 @@ RUN mkdir -p /var/run/sshd && \
     sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
     echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
     sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
-    ssh-keygen -A && \
     mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc
 
 ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so

diff --git a/dockerfiles/base/Dockerfile.rhel9.2 b/dockerfiles/base/Dockerfile.rhel9.2
@@ -53,6 +53,7 @@ RUN dnf install -y \
     wget \
     git \
     libffi-devel \
+    bzip2 \
     bzip2-devel \
     zlib-devel \
     mesa-libGL \
@@ -61,7 +62,8 @@ RUN dnf install -y \
     # update pkgs (except OS version) for resolving potentials CVEs
     dnf versionlock add redhat-release* && \
     dnf update -y && \
-    dnf clean all && rm -rf /var/cache/yum
+    dnf clean all && rm -rf /var/cache/yum && \
+    rm -f /etc/ssh/ssh_host_*_key*
 
 ENV PYTHON_VERSION=3.10
 COPY install-python310.sh .
@@ -71,15 +73,17 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
+ENV OPENMPI_VERSION=4.1.6
 ENV LIBFABRIC_VERSION="1.20.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
-ENV MPI_ROOT=/opt/amazon/openmpi
+ENV MPI_ROOT=/opt/habanalabs/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
 ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
 ENV OPAL_PREFIX=${MPI_ROOT}
 ENV MPICC=${MPI_ROOT}/bin/mpicc
 ENV RDMAV_FORK_SAFE=1
-ENV FI_EFA_USE_DEVICE_RDMA=1
+ENV FI_EFA_USE_DEVICE_RDMA=0
+ENV OMPI_MCA_btl=^openib
 
 RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
     echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
@@ -91,9 +95,9 @@ RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
 RUN update-crypto-policies --set DEFAULT:SHA1
 
 RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
-    habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
-    habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
-    habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
+        habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
+        habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
+        habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
     rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
     dnf clean all && rm -rf /var/cache/yum
 
@@ -111,14 +115,20 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}
 
+RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
+    tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
+    cd /tmp/openmpi-${OPENMPI_VERSION} && \
+    ./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
+    make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}
+
 RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
     unzip /tmp/main.zip -d /tmp && \
     cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.10 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
+RUN python3.10 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
@@ -130,7 +140,6 @@ RUN mkdir -p /var/run/sshd && \
     sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
     echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
     sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
-    ssh-keygen -A && \
     mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc
 
 ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so

diff --git a/dockerfiles/base/Dockerfile.rhel9.4 b/dockerfiles/base/Dockerfile.rhel9.4
@@ -51,28 +51,30 @@ RUN dnf install -y \
     lsof \
     python3-devel \
     openssh-clients \
-    openssl-1:3.0.7-27.el9 \
-    openssl-devel-1:3.0.7-27.el9 \
+    openssl-1:3.0.7-28.el9_4 \
+    openssl-devel-1:3.0.7-28.el9_4 \
     libjpeg-devel \
     openssh-server \
     lsb_release \
     wget \
     git \
     libffi-devel \
+    bzip2 \
     bzip2-devel \
     zlib-devel \
     mesa-libGL \
     iproute \
     python3.11 \
     python3.11-pip \
     python3.11-devel \
+    python3.11-rpm \
     ffmpeg-free \
-    perl-Net-SSLeay-1.92-2.el9 \
     python3-dnf-plugin-versionlock && \
     # update pkgs (except OS version) for resolving potentials CVEs
-    dnf versionlock add redhat-release* openssl* perl-Net-SSLeay && \
+    dnf versionlock add redhat-release* openssl* libcurl-minimal curl-minimal ima-evm-utils python3-rpm rpm* && \
     dnf update -y && \
-    dnf clean all && rm -rf /var/cache/yum
+    dnf clean all && rm -rf /var/cache/yum && \
+    rm -f /etc/ssh/ssh_host_*_key*
 
 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
     alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
@@ -84,15 +86,17 @@ RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
 COPY install_efa.sh .
 RUN ./install_efa.sh && rm install_efa.sh && rm -rf /etc/ld.so.conf.d/efa.conf /etc/profile.d/efa.sh
 
-ENV LIBFABRIC_VERSION="1.20.0"
+ENV OPENMPI_VERSION=4.1.6
+ENV LIBFABRIC_VERSION="1.22.0"
 ENV LIBFABRIC_ROOT="/opt/habanalabs/libfabric-${LIBFABRIC_VERSION}"
-ENV MPI_ROOT=/opt/amazon/openmpi
+ENV MPI_ROOT=/opt/habanalabs/openmpi
 ENV LD_LIBRARY_PATH=$LIBFABRIC_ROOT/lib:${MPI_ROOT}/lib:/usr/lib/habanalabs:$LD_LIBRARY_PATH
 ENV PATH=${LIBFABRIC_ROOT}/bin:${MPI_ROOT}/bin:$PATH
 ENV OPAL_PREFIX=${MPI_ROOT}
 ENV MPICC=${MPI_ROOT}/bin/mpicc
 ENV RDMAV_FORK_SAFE=1
-ENV FI_EFA_USE_DEVICE_RDMA=1
+ENV FI_EFA_USE_DEVICE_RDMA=0
+ENV OMPI_MCA_btl=^openib
 
 RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
     echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \
@@ -104,9 +108,9 @@ RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
 RUN update-crypto-policies --set DEFAULT:SHA1
 
 RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
-    habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
-    habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
-    habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
+        habanalabs-thunk-"$VERSION"-"$REVISION".el9 \
+        habanalabs-firmware-tools-"$VERSION"-"$REVISION".el9 \
+        habanalabs-graph-"$VERSION"-"$REVISION".el9 && \
     rm -f /etc/yum.repos.d/habanalabs.repo && rm -f /etc/yum.repos.d/habana.repo && rm -rf /tmp/* && \
     dnf clean all && rm -rf /var/cache/yum
 
@@ -124,14 +128,20 @@ RUN wget -nv -O /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 https://github.com/o
     ./configure --prefix=$LIBFABRIC_ROOT --enable-psm3-verbs --enable-verbs=yes --with-synapseai=/usr && \
     make && make install && cd / && rm -rf /tmp/libfabric-${LIBFABRIC_VERSION}.tar.bz2 /tmp/libfabric-${LIBFABRIC_VERSION}
 
+RUN wget -q -O /tmp/openmpi-${OPENMPI_VERSION}.tar.gz https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz && \
+    tar -xzf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz -C /tmp && \
+    cd /tmp/openmpi-${OPENMPI_VERSION} && \
+    ./configure --prefix=${MPI_ROOT} --with-libfabric=$LIBFABRIC_ROOT --with-verbs && \
+    make -j$(nproc) && make install && cd / && rm -rf /tmp/openmpi-${OPENMPI_VERSION}.tar.gz /tmp/openmpi-${OPENMPI_VERSION}
+
 RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archive/refs/heads/main.zip && \
     unzip /tmp/main.zip -d /tmp && \
     cd /tmp/hccl_ofi_wrapper-main && \
     make && cp -f libhccl_ofi_wrapper.so /usr/lib/habanalabs/libhccl_ofi_wrapper.so && \
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
-RUN python3.11 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
+RUN python3.11 -m pip install pip==24.2 setuptools==75.1.0 wheel==0.44.0
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
@@ -143,7 +153,6 @@ RUN mkdir -p /var/run/sshd && \
     sed -i 's/#\(ForwardAgent \).*/\1yes/g' /etc/ssh/ssh_config && \
     echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
     sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config && \
-    ssh-keygen -A && \
     mkdir -p /var/run/sshd && echo "/usr/sbin/sshd -p 3022" | tee -a ~/.bashrc
 
 ENV GC_KERNEL_PATH=/usr/lib/habanalabs/libtpc_kernels.so