From 424a31b1d5e5a9b790ccdb0f701c29346f68a08b Mon Sep 17 00:00:00 2001 From: alexsin368 <109180236+alexsin368@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:00:09 -0800 Subject: [PATCH 1/3] ChatQnA on Gaudi - fix tabs for build/pull, use TAG instead of specific version (#252) * update chatqna doc, use TAG instead of version Signed-off-by: alexsin368 * update instruction for building vLLM docker image with HPU Signed-off-by: alexsin368 --------- Signed-off-by: alexsin368 --- examples/ChatQnA/deploy/gaudi.md | 138 ++++++++++++++++--------------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/examples/ChatQnA/deploy/gaudi.md b/examples/ChatQnA/deploy/gaudi.md index aa59a165..926267fe 100644 --- a/examples/ChatQnA/deploy/gaudi.md +++ b/examples/ChatQnA/deploy/gaudi.md @@ -47,11 +47,13 @@ To summarize, Below is the flow of contents we will be covering in this tutorial First step is to clone the GenAIExamples and GenAIComps. GenAIComps are fundamental necessary components used to build examples you find in -GenAIExamples and deploy them as microservices. +GenAIExamples and deploy them as microservices. Also set the `TAG` +environment variable with the version. ``` git clone https://github.com/opea-project/GenAIComps.git git clone https://github.com/opea-project/GenAIExamples.git +export TAG=1.1 ``` The examples utilize model weights from HuggingFace and langchain. @@ -87,50 +89,51 @@ ChatQnA megaservice, and UI (conversational React UI is optional). In total, there are 8 required and an optional docker images. ### Build/Pull Microservice images -::::{tab-set} -:::{tab-item} Pull + +::::::{tab-set} + +:::::{tab-item} Pull :sync: Pull -To pull pre-built docker images on Docker Hub, proceed to the next step. To customize -your application, you can choose to build individual docker images for the microservices -before proceeding. -::: -:::{tab-item} Build +If you decide to pull the docker containers and not build them locally, +you can proceed to the next step where all the necessary containers will +be pulled in from dockerhub. + +::::: +:::::{tab-item} Build :sync: Build From within the `GenAIComps` folder, checkout the release tag. ``` cd GenAIComps -git checkout tags/v1.0 +git checkout tags/v${TAG} ``` -::: -:::: #### Build Dataprep Image ```bash -docker build --no-cache -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/Dockerfile . +docker build --no-cache -t opea/dataprep-redis:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/Dockerfile . ``` #### Build Embedding Image ```bash -docker build --no-cache -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile . +docker build --no-cache -t opea/embedding-tei:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/tei/langchain/Dockerfile . ``` #### Build Retriever Image ```bash -docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . +docker build --no-cache -t opea/retriever-redis:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile . ``` #### Build Rerank Image ```bash -docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile . +docker build --no-cache -t opea/reranking-tei:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/tei/Dockerfile . ``` -#### Build docker +#### Build LLM Image ::::{tab-set} @@ -139,12 +142,12 @@ docker build --no-cache -t opea/reranking-tei:latest --build-arg https_proxy=$ht Build vLLM docker image with hpu support ``` -docker build --no-cache -t opea/llm-vllm-hpu:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/vllm/langchain/dependency/Dockerfile.intel_hpu . +bash ./comps/llms/text-generation/vllm/langchain/dependency/build_docker_vllm.sh hpu ``` Build vLLM Microservice image ``` -docker build --no-cache -t opea/llm-vllm:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/vllm/langchain/Dockerfile . +docker build --no-cache -t opea/llm-vllm:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/vllm/langchain/Dockerfile . cd .. ``` ::: @@ -152,7 +155,7 @@ cd .. :sync: TGI ```bash -docker build --no-cache -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +docker build --no-cache -t opea/llm-tgi:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . ``` ::: :::: @@ -164,7 +167,7 @@ Since a TEI Gaudi Docker image hasn't been published, we'll need to build it fro ```bash git clone https://github.com/huggingface/tei-gaudi cd tei-gaudi/ -docker build --no-cache -f Dockerfile-hpu -t opea/tei-gaudi:latest . +docker build --no-cache -f Dockerfile-hpu -t opea/tei-gaudi:${TAG} . cd .. ``` @@ -183,12 +186,12 @@ Build the megaservice image for this use case ``` cd .. cd GenAIExamples -git checkout tags/v1.0 +git checkout tags/v1.1 cd ChatQnA ``` ```bash -docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +docker build --no-cache -t opea/chatqna:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . cd ../.. ``` @@ -198,7 +201,7 @@ If you want to enable guardrails microservice in the pipeline, please use the be ```bash cd GenAIExamples/ChatQnA/ -docker build --no-cache -t opea/chatqna-guardrails:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.guardrails . +docker build --no-cache -t opea/chatqna-guardrails:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.guardrails . cd ../.. ``` @@ -210,7 +213,7 @@ As mentioned, you can build 2 modes of UI ```bash cd GenAIExamples/ChatQnA/ui/ -docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +docker build --no-cache -t opea/chatqna-ui:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . cd ../../.. ``` @@ -219,43 +222,47 @@ If you want a conversational experience with chatqna megaservice. ```bash cd GenAIExamples/ChatQnA/ui/ -docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +docker build --no-cache -t opea/chatqna-conversation-ui:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . cd ../../.. ``` ### Sanity Check -Check if you have the below set of docker images, before moving on to the next step: +Check if you have the below set of docker images before moving on to the next step. The tags are +based on what you set the environment variable `TAG` to. ::::{tab-set} :::{tab-item} vllm :sync: vllm -* opea/dataprep-redis:latest -* opea/embedding-tei:latest -* opea/retriever-redis:latest -* opea/reranking-tei:latest -* opea/tei-gaudi:latest -* opea/chatqna:latest or opea/chatqna-guardrails:latest -* opea/chatqna:latest -* opea/chatqna-ui:latest -* opea/vllm:latest -* opea/llm-vllm:latest +* opea/dataprep-redis:${TAG} +* opea/embedding-tei:${TAG} +* opea/retriever-redis:${TAG} +* opea/reranking-tei:${TAG} +* opea/tei-gaudi:${TAG} +* opea/chatqna:${TAG} or opea/chatqna-guardrails:${TAG} +* opea/chatqna:${TAG} +* opea/chatqna-ui:${TAG} +* opea/vllm:${TAG} +* opea/llm-vllm:${TAG} ::: :::{tab-item} TGI :sync: TGI -* opea/dataprep-redis:latest -* opea/embedding-tei:latest -* opea/retriever-redis:latest -* opea/reranking-tei:latest -* opea/tei-gaudi:latest -* opea/chatqna:latest or opea/chatqna-guardrails:latest -* opea/chatqna-ui:latest -* opea/llm-tgi:latest +* opea/dataprep-redis:${TAG} +* opea/embedding-tei:${TAG} +* opea/retriever-redis:${TAG} +* opea/reranking-tei:${TAG} +* opea/tei-gaudi:${TAG} +* opea/chatqna:${TAG} or opea/chatqna-guardrails:${TAG} +* opea/chatqna-ui:${TAG} +* opea/llm-tgi:${TAG} ::: :::: +::::: +:::::: + ## Use Case Setup As mentioned the use case will use the following combination of the GenAIComps @@ -392,7 +399,8 @@ Check if all the containers launched via docker compose has started For example, the ChatQnA example starts 11 docker (services), check these docker containers are all running, i.e, all the containers `STATUS` are `Up` -To do a quick sanity check, try `docker ps -a` to see if all the containers are running +To do a quick sanity check, try `docker ps -a` to see if all the containers are running. +Note that `TAG` will be the value you set earlier. ::::{tab-set} @@ -400,15 +408,15 @@ To do a quick sanity check, try `docker ps -a` to see if all the containers are :sync: vllm ```bash CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -42c8d5ec67e9 opea/chatqna-ui:latest "docker-entrypoint.s…" About a minute ago Up About a minute 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp chatqna-gaudi-ui-server -7f7037a75f8b opea/chatqna:latest "python chatqna.py" About a minute ago Up About a minute 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp chatqna-gaudi-backend-server -4049c181da93 opea/embedding-tei:latest "python embedding_te…" About a minute ago Up About a minute 0.0.0.0:6000->6000/tcp, :::6000->6000/tcp embedding-tei-server -171816f0a789 opea/dataprep-redis:latest "python prepare_doc_…" About a minute ago Up About a minute 0.0.0.0:6007->6007/tcp, :::6007->6007/tcp dataprep-redis-server -10ee6dec7d37 opea/llm-vllm:latest "bash entrypoint.sh" About a minute ago Up About a minute 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp llm-vllm-gaudi-server -ce4e7802a371 opea/retriever-redis:latest "python retriever_re…" About a minute ago Up About a minute 0.0.0.0:7000->7000/tcp, :::7000->7000/tcp retriever-redis-server -be6cd2d0ea38 opea/reranking-tei:latest "python reranking_te…" About a minute ago Up About a minute 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp reranking-tei-gaudi-server -cc45ff032e8c opea/tei-gaudi:latest "text-embeddings-rou…" About a minute ago Up About a minute 0.0.0.0:8090->80/tcp, :::8090->80/tcp tei-embedding-gaudi-server -4969ec3aea02 opea/llm-vllm-hpu:latest "/bin/bash -c 'expor…" About a minute ago Up About a minute 0.0.0.0:8007->80/tcp, :::8007->80/tcp vllm-gaudi-server +42c8d5ec67e9 opea/chatqna-ui:${TAG} "docker-entrypoint.s…" About a minute ago Up About a minute 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp chatqna-gaudi-ui-server +7f7037a75f8b opea/chatqna:${TAG} "python chatqna.py" About a minute ago Up About a minute 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp chatqna-gaudi-backend-server +4049c181da93 opea/embedding-tei:${TAG} "python embedding_te…" About a minute ago Up About a minute 0.0.0.0:6000->6000/tcp, :::6000->6000/tcp embedding-tei-server +171816f0a789 opea/dataprep-redis:${TAG} "python prepare_doc_…" About a minute ago Up About a minute 0.0.0.0:6007->6007/tcp, :::6007->6007/tcp dataprep-redis-server +10ee6dec7d37 opea/llm-vllm:${TAG} "bash entrypoint.sh" About a minute ago Up About a minute 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp llm-vllm-gaudi-server +ce4e7802a371 opea/retriever-redis:${TAG} "python retriever_re…" About a minute ago Up About a minute 0.0.0.0:7000->7000/tcp, :::7000->7000/tcp retriever-redis-server +be6cd2d0ea38 opea/reranking-tei:${TAG} "python reranking_te…" About a minute ago Up About a minute 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp reranking-tei-gaudi-server +cc45ff032e8c opea/tei-gaudi:${TAG} "text-embeddings-rou…" About a minute ago Up About a minute 0.0.0.0:8090->80/tcp, :::8090->80/tcp tei-embedding-gaudi-server +4969ec3aea02 opea/llm-vllm-hpu:${TAG} "/bin/bash -c 'expor…" About a minute ago Up About a minute 0.0.0.0:8007->80/tcp, :::8007->80/tcp vllm-gaudi-server 0657cb66df78 redis/redis-stack:7.2.0-v9 "/entrypoint.sh" About a minute ago Up About a minute 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp redis-vector-db 684d3e9d204a ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 "text-embeddings-rou…" About a minute ago Up About a minute 0.0.0.0:8808->80/tcp, :::8808->80/tcp tei-reranking-gaudi-server ``` @@ -418,15 +426,15 @@ cc45ff032e8c opea/tei-gaudi:latest "text-emb :sync: TGI ```bash CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -0355d705484a opea/chatqna-ui:latest "docker-entrypoint.s…" 2 minutes ago Up 2 minutes 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp chatqna-gaudi-ui-server -29a7a43abcef opea/chatqna:latest "python chatqna.py" 2 minutes ago Up 2 minutes 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp chatqna-gaudi-backend-server -1eb6f5ad6f85 opea/llm-tgi:latest "bash entrypoint.sh" 2 minutes ago Up 2 minutes 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp llm-tgi-gaudi-server -ad27729caf68 opea/reranking-tei:latest "python reranking_te…" 2 minutes ago Up 2 minutes 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp reranking-tei-gaudi-server -84f02cf2a904 opea/dataprep-redis:latest "python prepare_doc_…" 2 minutes ago Up 2 minutes 0.0.0.0:6007->6007/tcp, :::6007->6007/tcp dataprep-redis-server -367459f6e65b opea/embedding-tei:latest "python embedding_te…" 2 minutes ago Up 2 minutes 0.0.0.0:6000->6000/tcp, :::6000->6000/tcp embedding-tei-server -8c78cde9f588 opea/retriever-redis:latest "python retriever_re…" 2 minutes ago Up 2 minutes 0.0.0.0:7000->7000/tcp, :::7000->7000/tcp retriever-redis-server +0355d705484a opea/chatqna-ui:${TAG} "docker-entrypoint.s…" 2 minutes ago Up 2 minutes 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp chatqna-gaudi-ui-server +29a7a43abcef opea/chatqna:${TAG} "python chatqna.py" 2 minutes ago Up 2 minutes 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp chatqna-gaudi-backend-server +1eb6f5ad6f85 opea/llm-tgi:${TAG} "bash entrypoint.sh" 2 minutes ago Up 2 minutes 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp llm-tgi-gaudi-server +ad27729caf68 opea/reranking-tei:${TAG} "python reranking_te…" 2 minutes ago Up 2 minutes 0.0.0.0:8000->8000/tcp, :::8000->8000/tcp reranking-tei-gaudi-server +84f02cf2a904 opea/dataprep-redis:${TAG} "python prepare_doc_…" 2 minutes ago Up 2 minutes 0.0.0.0:6007->6007/tcp, :::6007->6007/tcp dataprep-redis-server +367459f6e65b opea/embedding-tei:${TAG} "python embedding_te…" 2 minutes ago Up 2 minutes 0.0.0.0:6000->6000/tcp, :::6000->6000/tcp embedding-tei-server +8c78cde9f588 opea/retriever-redis:${TAG} "python retriever_re…" 2 minutes ago Up 2 minutes 0.0.0.0:7000->7000/tcp, :::7000->7000/tcp retriever-redis-server fa80772de92c ghcr.io/huggingface/tgi-gaudi:2.0.1 "text-generation-lau…" 2 minutes ago Up 2 minutes 0.0.0.0:8005->80/tcp, :::8005->80/tcp tgi-gaudi-server -581687a2cc1a opea/tei-gaudi:latest "text-embeddings-rou…" 2 minutes ago Up 2 minutes 0.0.0.0:8090->80/tcp, :::8090->80/tcp tei-embedding-gaudi-server +581687a2cc1a opea/tei-gaudi:${TAG} "text-embeddings-rou…" 2 minutes ago Up 2 minutes 0.0.0.0:8090->80/tcp, :::8090->80/tcp tei-embedding-gaudi-server c59178629901 redis/redis-stack:7.2.0-v9 "/entrypoint.sh" 2 minutes ago Up 2 minutes 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp redis-vector-db 5c3a78144498 ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 "text-embeddings-rou…" 2 minutes ago Up 2 minutes 0.0.0.0:8808->80/tcp, :::8808->80/tcp tei-reranking-gaudi-server ``` @@ -796,7 +804,7 @@ curl http://${host_ip}:9090/v1/guardrails\ To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the compose.yaml file as shown below: ```bash chaqna-gaudi-ui-server: - image: opea/chatqna-ui:latest + image: opea/chatqna-ui:${TAG} ... ports: - "80:5173" @@ -806,7 +814,7 @@ To access the frontend, open the following URL in your browser: http://{host_ip} To access the Conversational UI (react based) frontend, modify the UI service in the compose.yaml file. Replace chaqna-gaudi-ui-server service with the chatqna-gaudi-conversation-ui-server service as per the config below: ```bash chaqna-gaudi-conversation-ui-server: - image: opea/chatqna-conversation-ui:latest + image: opea/chatqna-conversation-ui:${TAG} container_name: chatqna-gaudi-conversation-ui-server environment: - APP_BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT} @@ -821,7 +829,7 @@ chaqna-gaudi-conversation-ui-server: Once the services are up, open the following URL in your browser: http://{host_ip}:5174. By default, the UI runs on port 80 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the compose.yaml file as shown below: ``` chaqna-gaudi-conversation-ui-server: - image: opea/chatqna-conversation-ui:latest + image: opea/chatqna-conversation-ui:${TAG} ... ports: - "80:80" From da26f2734b49c3f043618f90b1fad8b88a4b496f Mon Sep 17 00:00:00 2001 From: Neo Zhang Jianyu Date: Mon, 18 Nov 2024 13:58:13 +0800 Subject: [PATCH 2/3] Support multiple cores to accelerate build. (#253) * support multiple cores * use auto for all cores --------- Co-authored-by: ZhangJianyu --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2026cc86..701baac2 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ else endif # You can set these variables from the command line. -SPHINXOPTS ?= -q +SPHINXOPTS ?= -q -j auto SPHINXBUILD = sphinx-build SPHINXPROJ = "OPEA Project" BUILDDIR ?= _build From 1ec71a5a617b540db67747bf8f9f8e527850514a Mon Sep 17 00:00:00 2001 From: alexsin368 <109180236+alexsin368@users.noreply.github.com> Date: Mon, 18 Nov 2024 18:10:33 -0800 Subject: [PATCH 3/3] Add codegen sample guide for Gaudi deployment (#248) * adding codegen sample guide for gaudi deployment Signed-off-by: alexsin368 * update with TAG instead of version number Signed-off-by: alexsin368 * remove mention of vllm Signed-off-by: alexsin368 * adding codegen sample guide for gaudi deployment Signed-off-by: alexsin368 * update with TAG instead of version number Signed-off-by: alexsin368 * remove mention of vllm Signed-off-by: alexsin368 * fix typos, add link to ITAC, attempt to fix IO flow diagram link issue Signed-off-by: alexsin368 * modify and add index.rst files Signed-off-by: alexsin368 * make text output to not think it's a hyperlink Signed-off-by: alexsin368 --------- Signed-off-by: alexsin368 Co-authored-by: Ying Hu --- examples/CodeGen/deploy/gaudi.md | 375 ++++++++++++++++++++++++++++++ examples/CodeGen/deploy/index.rst | 14 ++ examples/index.rst | 2 + 3 files changed, 391 insertions(+) create mode 100644 examples/CodeGen/deploy/gaudi.md create mode 100644 examples/CodeGen/deploy/index.rst diff --git a/examples/CodeGen/deploy/gaudi.md b/examples/CodeGen/deploy/gaudi.md new file mode 100644 index 00000000..d4dfafa9 --- /dev/null +++ b/examples/CodeGen/deploy/gaudi.md @@ -0,0 +1,375 @@ +# Single node on-prem deployment with TGI on Gaudi AI Accelerator + +This deployment section covers single-node on-prem deployment of the CodeGen +example with OPEA comps to deploy using the TGI service. We will be showcasing how +to build an e2e CodeGen solution with the CodeLlama-7b-hf model, deployed on Intel® +Tiber™ AI Cloud ([ITAC](https://www.intel.com/content/www/us/en/developer/tools/tiber/ai-cloud.html)). +To quickly learn about OPEA in just 5 minutes and set up the required hardware and software, +please follow the instructions in the [Getting Started](https://opea-project.github.io/latest/getting-started/README.html) +section. If you do not have an ITAC instance or the hardware is not supported in the ITAC yet, you can still run this on-prem. + +## Overview + +The CodeGen use case uses a single microservice called LLM. In this tutorial, we +will walk through the steps on how to enable it from OPEA GenAIComps to deploy on +a single node TGI megaservice solution. + +The solution is aimed to show how to use the CodeLlama-7b-hf model on the Intel® +Gaudi® AI Accelerator. We will go through how to setup docker containers to start +the microservice and megaservice. The solution will then take text input as the +prompt and generate code accordingly. It is deployed with a UI with 2 modes to +choose from: + +1. Svelte-Based UI +2. React-Based UI + +The React-based UI is optional, but this feature is supported in this example if you +are interested in using it. + +Below is the list of content we will be covering in this tutorial: + +1. Prerequisites +2. Prepare (Building / Pulling) Docker images +3. Use case setup +4. Deploy the use case +5. Interacting with CodeGen deployment + +## Prerequisites + +The first step is to clone the GenAIExamples and GenAIComps. GenAIComps are +fundamental necessary components used to build examples you find in +GenAIExamples and deploy them as microservices. Also set the `TAG` +environment variable with the version. + +```bash +git clone https://github.com/opea-project/GenAIComps.git +git clone https://github.com/opea-project/GenAIExamples.git +export TAG=1.1 +``` + +The examples utilize model weights from HuggingFace and langchain. + +Setup your [HuggingFace](https://huggingface.co/) account and generate +[user access token](https://huggingface.co/docs/transformers.js/en/guides/private#step-1-generating-a-user-access-token). + +Setup the HuggingFace token +``` +export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" +``` + +Additionally, if you plan to use the default model CodeLlama-7b-hf, you will +need to [request access](https://huggingface.co/meta-llama/CodeLlama-7b-hf) from HuggingFace. + +The example requires you to set the `host_ip` to deploy the microservices on +endpoint enabled with ports. Set the host_ip env variable +``` +export host_ip=$(hostname -I | awk '{print $1}') +``` + +Make sure to setup Proxies if you are behind a firewall +``` +export no_proxy=${your_no_proxy},$host_ip +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +``` + +## Prepare (Building / Pulling) Docker images + +This step will involve building/pulling relevant docker +images with step-by-step process along with sanity check in the end. For +CodeGen, the following docker images will be needed: LLM with TGI. +Additionally, you will need to build docker images for the +CodeGen megaservice, and UI (React UI is optional). In total, +there are **3 required docker images** and an optional docker image. + +### Build/Pull Microservice image + +::::::{tab-set} + +:::::{tab-item} Pull +:sync: Pull + +If you decide to pull the docker containers and not build them locally, +you can proceed to the next step where all the necessary containers will +be pulled in from dockerhub. + +::::: +:::::{tab-item} Build +:sync: Build + +From within the `GenAIComps` folder, checkout the release tag. +``` +cd GenAIComps +git checkout tags/v${TAG} +``` + +#### Build LLM Image + +```bash +docker build --no-cache -t opea/llm-tgi:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile . +``` + +### Build Mega Service images + +The Megaservice is a pipeline that channels data through different +microservices, each performing varied tasks. The LLM microservice and +flow of data are defined in the `codegen.py` file. You can also add or +remove microservices and customize the megaservice to suit your needs. + +Build the megaservice image for this use case + +```bash +cd .. +cd GenAIExamples +git checkout tags/v${TAG} +cd CodeGen +``` + +```bash +docker build --no-cache -t opea/codegen:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +cd ../.. +``` + +### Build the UI Image + +You can build 2 modes of UI + +*Svelte UI* + +```bash +cd GenAIExamples/CodeGen/ui/ +docker build --no-cache -t opea/codegen-ui:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd ../../.. +``` + +*React UI (Optional)* +If you want a React-based frontend. + +```bash +cd GenAIExamples/CodeGen/ui/ +docker build --no-cache -t opea/codegen-react-ui:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +cd ../../.. +``` + +### Sanity Check +Check if you have the following set of docker images by running the command `docker images` before moving on to the next step. +The tags are based on what you set the environment variable `TAG` to. + +* `opea/llm-tgi:${TAG}` +* `opea/codegen:${TAG}` +* `opea/codegen-ui:${TAG}` +* `opea/codegen-react-ui:${TAG}` (optional) + +::::: +:::::: + +## Use Case Setup + +The use case will use the following combination of GenAIComps and tools + +|Use Case Components | Tools | Model | Service Type | +|---------------- |--------------|-----------------------------|-------| +|LLM | TGI | meta-llama/CodeLlama-7b-hf | OPEA Microservice | +|UI | | NA | Gateway Service | + +Tools and models mentioned in the table are configurable either through the +environment variables or `compose.yaml` file. + +Set the necessary environment variables to setup the use case by running the `set_env.sh` script. +Here is where the environment variable `LLM_MODEL_ID` is set, and you can change it to another model +by specifying the HuggingFace model card ID. + +```bash +cd GenAIExamples/CodeGen/docker_compose/ +source ./set_env.sh +cd ../../.. +``` + +## Deploy the Use Case + +In this tutorial, we will be deploying via docker compose with the provided +YAML file. The docker compose instructions should be starting all the +above mentioned services as containers. + +```bash +cd GenAIExamples/CodeGen/docker_compose/intel/hpu/gaudi +docker compose up -d +``` + + +### Checks to Ensure the Services are Running +#### Check Startup and Env Variables +Check the startup log by running `docker compose logs` to ensure there are no errors. +The warning messages print out the variables if they are **NOT** set. + +Here are some sample messages if proxy environment variables are not set: + + WARN[0000] The "no_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "https_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "http_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "no_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "https_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "http_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "no_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "http_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "https_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "no_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "http_proxy" variable is not set. Defaulting to a blank string. + WARN[0000] The "https_proxy" variable is not set. Defaulting to a blank string. + +#### Check the Container Status +Check if all the containers launched via docker compose have started. + +The CodeGen example starts 4 docker containers. Check that these docker +containers are all running, i.e, all the containers `STATUS` are `Up`. +You can do this with the `docker ps -a` command. + +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +bbd235074c3d opea/codegen-ui:${TAG} "docker-entrypoint.s…" About a minute ago Up About a minute 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp codegen-gaudi-ui-server +8d3872ca66fa opea/codegen:${TAG} "python codegen.py" About a minute ago Up About a minute 0.0.0.0:7778->7778/tcp, :::7778->7778/tcp codegen-gaudi-backend-server +b9fc39f51cdb opea/llm-tgi:${TAG} "bash entrypoint.sh" About a minute ago Up About a minute 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp llm-tgi-gaudi-server +39994e007f15 ghcr.io/huggingface/tgi-gaudi:2.0.1 "text-generation-lau…" About a minute ago Up About a minute 0.0.0.0:8028->80/tcp, :::8028->80/tcp tgi-gaudi-server +``` + +## Interacting with CodeGen for Deployment + +This section will walk you through the different ways to interact with +the microservices deployed. After a couple minutes, rerun `docker ps -a` +to ensure all the docker containers are still up and running. Then proceed +to validate each microservice and megaservice. + +### TGI Service + +```bash +curl http://${host_ip}:8028/generate \ + -X POST \ + -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":256, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +Here is the output: + +``` +{"generated_text":"\n\nIO iflow diagram:\n\n!\[IO flow diagram(s)\]\(TodoList.iflow.svg\)\n\n### TDD Kata walkthrough\n\n1. Start with a user story. We will add story tests later. In this case, we'll choose a story about adding a TODO:\n ```ruby\n as a user,\n i want to add a todo,\n so that i can get a todo list.\n\n conformance:\n - a new todo is added to the list\n - if the todo text is empty, raise an exception\n ```\n\n1. Write the first test:\n ```ruby\n feature Testing the addition of a todo to the list\n\n given a todo list empty list\n when a user adds a todo\n the todo should be added to the list\n\n inputs:\n when_values: [[\"A\"]]\n\n output validations:\n - todo_list contains { text:\"A\" }\n ```\n\n1. Write the first step implementation in any programming language you like. In this case, we will choose Ruby:\n ```ruby\n def add_"} +``` + +### LLM Microservice + +```bash +curl http://${host_ip}:9000/v1/chat/completions\ + -X POST \ + -d '{"query":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","max_tokens":256,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \ + -H 'Content-Type: application/json' +``` + +The output is given one character at a time. It is too long to show +here but the last item will be +``` +data: [DONE] +``` + +### MegaService + +```bash +curl http://${host_ip}:7778/v1/codegen -H "Content-Type: application/json" -d '{ + "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception." + }' +``` + +The output is given one character at a time. It is too long to show +here but the last item will be +``` +data: [DONE] +``` + +## Launch UI +### Svelte UI +To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: +```bash + codegen-gaudi-ui-server: + image: ${REGISTRY:-opea}/codegen-ui:${TAG:-latest} + ... + ports: + - "5173:5173" +``` + +### React-Based UI (Optional) +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `codegen-gaudi-ui-server` service with the codegen-gaudi-react-ui-server service as per the config below: +```bash +codegen-gaudi-react-ui-server: + image: ${REGISTRY:-opea}/codegen-react-ui:${TAG:-latest} + container_name: codegen-gaudi-react-ui-server + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - APP_CODE_GEN_URL=${BACKEND_SERVICE_ENDPOINT} + depends_on: + - codegen-gaudi-backend-server + ports: + - "5174:80" + ipc: host + restart: always +``` +Once the services are up, open the following URL in your browser: http://{host_ip}:5174. By default, the UI runs on port 80 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: +```bash + codegen-gaudi-react-ui-server: + image: ${REGISTRY:-opea}/codegen-react-ui:${TAG:-latest} + ... + ports: + - "80:80" +``` + +## Check Docker Container Logs + +You can check the log of a container by running this command: + +```bash +docker logs -t +``` + +You can also check the overall logs with the following command, where the +`compose.yaml` is the megaservice docker-compose configuration file. + +Assumming you are still in this directory `GenAIExamples/CodeGen/docker_compose/intel/hpu/gaudi`, +run the following command to check the logs: +```bash +docker compose -f compose.yaml logs +``` + +View the docker input parameters in `./CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml` + +```yaml + tgi-service: + image: ghcr.io/huggingface/tgi-gaudi:2.0.1 + container_name: tgi-gaudi-server + ports: + - "8028:80" + volumes: + - "./data:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + runtime: habana + cap_add: + - SYS_NICE + ipc: host + command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 +``` + +The input `--model-id` is `${LLM_MODEL_ID}`. Ensure the environment variable `LLM_MODEL_ID` +is set and spelled correctly. Check spelling. Whenever this is changed, restart the containers to use +the newly selected model. + + +## Stop the services + +Once you are done with the entire pipeline and wish to stop and remove all the containers, use the command below: +``` +docker compose down +``` diff --git a/examples/CodeGen/deploy/index.rst b/examples/CodeGen/deploy/index.rst new file mode 100644 index 00000000..ac0a37d0 --- /dev/null +++ b/examples/CodeGen/deploy/index.rst @@ -0,0 +1,14 @@ +.. _codegen-example-deployment: + +CodeGen Example Deployment Options +################################### + +Here are some deployment options, depending on your hardware and environment: + +Single Node +*********** + +.. toctree:: + :maxdepth: 1 + + Gaudi AI Accelerator \ No newline at end of file diff --git a/examples/index.rst b/examples/index.rst index 6524dc1d..283693f3 100644 --- a/examples/index.rst +++ b/examples/index.rst @@ -12,6 +12,8 @@ GenAIExamples are designed to give developers an easy entry into generative AI, ChatQnA/deploy/index AgentQnA/AgentQnA_Guide AgentQnA/deploy/index + CodeGen/deploy/gaudi.md + CodeGen/deploy/index ----