From 2477b05e7ec9e9938f4c41ff9b171f911d8b9ff4 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 2 May 2023 16:23:42 +0100 Subject: [PATCH] Add runtime example Signed-off-by: Martin Hickey --- examples/text-sentiment/README.md | 81 +++++++++++++++++++ examples/text-sentiment/client.py | 27 +++++++ .../models/text_sentiment/config.yml | 4 + examples/text-sentiment/requirements.txt | 7 ++ examples/text-sentiment/start_runtime.py | 12 +++ .../text-sentiment/text_sentiment/__init__.py | 12 +++ .../text-sentiment/text_sentiment/config.yml | 6 ++ .../text_sentiment/data_model/__init__.py | 2 + .../data_model/classification.py | 25 ++++++ .../text_sentiment/runtime_model/__init__.py | 2 + .../text_sentiment/runtime_model/hf_block.py | 71 ++++++++++++++++ 11 files changed, 249 insertions(+) create mode 100644 examples/text-sentiment/README.md create mode 100644 examples/text-sentiment/client.py create mode 100644 examples/text-sentiment/models/text_sentiment/config.yml create mode 100644 examples/text-sentiment/requirements.txt create mode 100644 examples/text-sentiment/start_runtime.py create mode 100644 examples/text-sentiment/text_sentiment/__init__.py create mode 100644 examples/text-sentiment/text_sentiment/config.yml create mode 100644 examples/text-sentiment/text_sentiment/data_model/__init__.py create mode 100644 examples/text-sentiment/text_sentiment/data_model/classification.py create mode 100644 examples/text-sentiment/text_sentiment/runtime_model/__init__.py create mode 100644 examples/text-sentiment/text_sentiment/runtime_model/hf_block.py diff --git a/examples/text-sentiment/README.md b/examples/text-sentiment/README.md new file mode 100644 index 000000000..63d261159 --- /dev/null +++ b/examples/text-sentiment/README.md @@ -0,0 +1,81 @@ +# Text Sentiment Analysis Example + +This example uses the [HuggingFace DistilBERT base uncased finetuned SST-2](https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english) AI model to perform text sentiment analysis. The Caikit runtime loads the model and serves it so that it can be inferred or called. + +## Before Starting + +The following tools are required: + +- [python](https://www.python.org) (v3.8+) +- [pip](https://pypi.org/project/pip/) (v23.0+) + +**Note: Before installing dependencie and to avoid conflicts in your environment, it is advisable to use a [virtual environment(venv)](https://docs.python.org/3/library/venv.html).** + +Install the dependencies: `pip install -r requirements.txt` + +## Running the Caikit runtime + +In one terminal, start the runtime server: + +```shell +python3 start_runtime.py +``` + +You should see output similar to the following: + +```command +$ python3 start_runtime.py + + is still in the BETA phase and subject to change! +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: text_sentiment", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:52.808812"} +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: caikit.interfaces.common", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:52.809406"} +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: caikit.interfaces.runtime", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:52.809565"} +[…] +{"channel": "MODEL-LOADER", "exception": null, "level": "info", "log_code": "", "message": "Loading model 'text_sentiment'", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:52.826657"} +{"channel": "MDLMNG", "exception": null, "level": "warning", "log_code": "", "message": "No backend configured! Trying to configure using default config file.", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:52.827742"} +No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english). +Using a pipeline without specifying a model name and revision in production is not recommended. +[…] +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: text_sentiment", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.929756"} +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: caikit.interfaces.common", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.929814"} +{"channel": "COM-LIB-INIT", "exception": null, "level": "info", "log_code": "", "message": "Loading service module: caikit.interfaces.runtime", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.929858"} +{"channel": "GP-SERVICR-I", "exception": null, "level": "info", "log_code": "", "message": "Validated Caikit Library CDM successfully", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.929942"} +{"channel": "GP-SERVICR-I", "exception": null, "level": "info", "log_code": "", "message": "Constructed inference service for library: text_sentiment, version: unknown", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.930734"} +{"channel": "SERVER-WRAPR", "exception": null, "level": "info", "log_code": "", "message": "Intercepting RPC method /caikit.runtime.HfTextsentiment.HfTextsentimentService/HfBlockPredict", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.930786"} +{"channel": "SERVER-WRAPR", "exception": null, "level": "info", "log_code": "", "message": "Wrapping safe rpc for Predict", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.931424"} +{"channel": "SERVER-WRAPR", "exception": null, "level": "info", "log_code": "", "message": "Re-routing RPC /caikit.runtime.HfTextsentiment.HfTextsentimentService/HfBlockPredict from . at 0x7fce01f660d0> to .safe_rpc_call at 0x7fce02144670>", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.931479"} +{"channel": "SERVER-WRAPR", "exception": null, "level": "info", "log_code": "", "message": "Interception of service caikit.runtime.HfTextsentiment.HfTextsentimentService complete", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.931530"} +[…] + +{"channel": "GRPC-SERVR", "exception": null, "level": "info", "log_code": "", "message": "Running in insecure mode", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.936511"} +{"channel": "GRPC-SERVR", "exception": null, "level": "info", "log_code": "", "message": "Caikit Runtime is serving on port: 8085 with thread pool size: 5", "num_indent": 0, "thread_id": 8605140480, "timestamp": "2023-05-02T11:42:53.938054"} +``` + +## Inferring the Served Model + +In another terminal, run the client code: + +```shell +python3 client.py +``` + +The client code calls the model and queries it for sentiment analysis on a 2 different pieces of text. + +You should see output similar to the following: + +```command +$ python3 client.py + + is still in the BETA phase and subject to change! +Text: I am not feeling well today! +RESPONSE: classes { + class_name: "NEGATIVE" + confidence: 0.99977594614028931 +} + +Text: Today is a nice sunny day +RESPONSE: classes { + class_name: "POSITIVE" + confidence: 0.999869704246521 +} +``` diff --git a/examples/text-sentiment/client.py b/examples/text-sentiment/client.py new file mode 100644 index 000000000..c1bd962c5 --- /dev/null +++ b/examples/text-sentiment/client.py @@ -0,0 +1,27 @@ +import grpc +from caikit.runtime.service_factory import ServicePackageFactory + +from text_sentiment.data_model import TextInput + +inference_service = ServicePackageFactory().get_service_package( + ServicePackageFactory.ServiceType.INFERENCE, + ServicePackageFactory.ServiceSource.GENERATED, +) + +port = 8085 +channel = grpc.insecure_channel(f"localhost:{port}") + +client_stub = inference_service.stub_class(channel) + +# print(dir(client_stub)) + +for text in ["I am not feeling well today!", "Today is a nice sunny day"]: + input_text_proto = TextInput(text=text).to_proto() + request = inference_service.messages.HfBlockRequest(text_input=input_text_proto) + response = client_stub.HfBlockPredict( + request, metadata=[("mm-model-id", "text_sentiment")] + ) + print("Text:", text) + print("RESPONSE:", response) + + diff --git a/examples/text-sentiment/models/text_sentiment/config.yml b/examples/text-sentiment/models/text_sentiment/config.yml new file mode 100644 index 000000000..288524cae --- /dev/null +++ b/examples/text-sentiment/models/text_sentiment/config.yml @@ -0,0 +1,4 @@ +block_id: 8f72161-c0e4-49b0-8fd0-7587b3017a35 +name: HuggingFaceSentimentBlock +version: 0.0.1 + diff --git a/examples/text-sentiment/requirements.txt b/examples/text-sentiment/requirements.txt new file mode 100644 index 000000000..3a1e1a1c0 --- /dev/null +++ b/examples/text-sentiment/requirements.txt @@ -0,0 +1,7 @@ +caikit + +# Only needed for HuggingFace +scipy +torch +transformers~=4.27.2 + diff --git a/examples/text-sentiment/start_runtime.py b/examples/text-sentiment/start_runtime.py new file mode 100644 index 000000000..1605206bf --- /dev/null +++ b/examples/text-sentiment/start_runtime.py @@ -0,0 +1,12 @@ +from os import path +import sys +import alog + +sys.path.append(path.abspath(path.join(path.dirname(__file__), "../"))) # Here we assume this file is at the same level of requirements.txt +import text_sentiment + +alog.configure(default_level="debug") + +from caikit.runtime import grpc_server +grpc_server.main() + diff --git a/examples/text-sentiment/text_sentiment/__init__.py b/examples/text-sentiment/text_sentiment/__init__.py new file mode 100644 index 000000000..487987a37 --- /dev/null +++ b/examples/text-sentiment/text_sentiment/__init__.py @@ -0,0 +1,12 @@ +import os + +from . import data_model, runtime_model +import caikit + +# Give the path to the `config.yml` +CONFIG_PATH = os.path.realpath( + os.path.join(os.path.dirname(__file__), "config.yml") +) + +caikit.configure(CONFIG_PATH) + diff --git a/examples/text-sentiment/text_sentiment/config.yml b/examples/text-sentiment/text_sentiment/config.yml new file mode 100644 index 000000000..d4f770547 --- /dev/null +++ b/examples/text-sentiment/text_sentiment/config.yml @@ -0,0 +1,6 @@ +runtime: + library: text_sentiment + service_generation: + primitive_data_model_types: + - "text_sentiment.data_model.classification.TextInput" + diff --git a/examples/text-sentiment/text_sentiment/data_model/__init__.py b/examples/text-sentiment/text_sentiment/data_model/__init__.py new file mode 100644 index 000000000..8d88e071f --- /dev/null +++ b/examples/text-sentiment/text_sentiment/data_model/__init__.py @@ -0,0 +1,2 @@ +from .classification import ClassificationPrediction, ClassInfo, TextInput + diff --git a/examples/text-sentiment/text_sentiment/data_model/classification.py b/examples/text-sentiment/text_sentiment/data_model/classification.py new file mode 100644 index 000000000..25ba5b85b --- /dev/null +++ b/examples/text-sentiment/text_sentiment/data_model/classification.py @@ -0,0 +1,25 @@ +from caikit.core.data_model import dataobject + +@dataobject( + package="text_sentiment.data_model", + schema={ + "class_name": str, # (required) Predicted relevant class name + "confidence": float, # (required) The confidence-like score of this prediction in [0, 1] + }, +) +class ClassInfo: + """A single classification prediction.""" + +@dataobject( + package="text_sentiment.data_model", + schema={"classes": {"elements": ClassInfo}}, +) +class ClassificationPrediction: + """The result of a classification prediction.""" + +@dataobject(package="text_sentiment.data_model", schema={"text": str}) +class TextInput: + """A sample `domain primitive` input type for this library. + The analog to a `Raw Document` for the `Natural Language Processing` domain.""" + + diff --git a/examples/text-sentiment/text_sentiment/runtime_model/__init__.py b/examples/text-sentiment/text_sentiment/runtime_model/__init__.py new file mode 100644 index 000000000..e14642fe9 --- /dev/null +++ b/examples/text-sentiment/text_sentiment/runtime_model/__init__.py @@ -0,0 +1,2 @@ +from .hf_block import HuggingFaceSentimentBlock + diff --git a/examples/text-sentiment/text_sentiment/runtime_model/hf_block.py b/examples/text-sentiment/text_sentiment/runtime_model/hf_block.py new file mode 100644 index 000000000..716254670 --- /dev/null +++ b/examples/text-sentiment/text_sentiment/runtime_model/hf_block.py @@ -0,0 +1,71 @@ +import os + +from caikit.core import BlockBase, ModuleLoader, ModuleSaver, block +from transformers import pipeline + +from text_sentiment.data_model.classification import ClassificationPrediction, ClassInfo, TextInput + +@block("8f72161-c0e4-49b0-8fd0-7587b3017a35", "HuggingFaceSentimentBlock", "0.0.1") +class HuggingFaceSentimentBlock(BlockBase): + """Class to wrap sentiment analysis pipeline from HuggingFace""" + + def __init__(self, model_path) -> None: + super().__init__() + loader = ModuleLoader(model_path) + config = loader.config + model = pipeline(model=config.hf_artifact_path, task="sentiment-analysis") + self.sentiment_pipeline = model + + def run(self, text_input: TextInput) -> ClassificationPrediction: + """Run HF sentiment analysis + Args: + text_input: TextInput + Returns: + ClassificationPrediction: predicted classes with their confidence score. + """ + raw_results = self.sentiment_pipeline([text_input.text]) + + class_info = [] + for result in raw_results: + class_info.append( + ClassInfo(class_name=result["label"], confidence=result["score"]) + ) + return ClassificationPrediction(class_info) + + @classmethod + def bootstrap(cls, model_path="distilbert-base-uncased-finetuned-sst-2-english"): + """Load a HuggingFace based caikit model + Args: + model_path: str + Path to HugginFace model + Returns: + HuggingFaceModel + """ + return cls(model_path) + + def save(self, model_path, **kwargs): + module_saver = ModuleSaver( + self, + model_path=model_path, + ) + + # Extract object to be saved + with module_saver: + # Make the directory to save model artifacts + rel_path, _ = module_saver.add_dir("hf_model") + save_path = os.path.join(model_path, rel_path) + self.sentiment_pipeline.save_pretrained(save_path) + module_saver.update_config({"hf_artifact_path": rel_path}) + + # this is how you load the model, if you have a caikit model + @classmethod + def load(cls, model_path): + """Load a HuggingFace based caikit model + Args: + model_path: str + Path to HuggingFace model + Returns: + HuggingFaceModel + """ + return cls(model_path) +