From e2b9624c4a01277bf9176497bbfb2ef10839e76b Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Tue, 30 Jul 2024 23:58:40 -0400 Subject: [PATCH 01/12] Introduce Segment Anything 2.0 for GPU only. Segment Anything 2.0 require to compile a .cu file with nvcc at build time. Hence, a cuda devel baseImage is required to build the nuclio container. --- README.md | 1 + .../sam2/nuclio/function-gpu.yaml | 67 +++++++++++++++++++ .../facebookresearch/sam2/nuclio/main.py | 42 ++++++++++++ .../sam2/nuclio/model_handler.py | 28 ++++++++ 4 files changed, 138 insertions(+) create mode 100644 serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml create mode 100644 serverless/pytorch/facebookresearch/sam2/nuclio/main.py create mode 100644 serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py diff --git a/README.md b/README.md index 61df75d6e44b..d45bdb2254bf 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,7 @@ up to 10x. Here is a list of the algorithms we support, and the platforms they c | Name | Type | Framework | CPU | GPU | | ------------------------------------------------------------------------------------------------------- | ---------- | ---------- | --- | --- | +| [Segment Anything 2.0](/serverless/pytorch/facebookresearch/sam2/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | | [Segment Anything](/serverless/pytorch/facebookresearch/sam/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | | [Deep Extreme Cut](/serverless/openvino/dextr/nuclio) | interactor | OpenVINO | ✔️ | | | [Faster RCNN](/serverless/openvino/omz/public/faster_rcnn_inception_resnet_v2_atrous_coco/nuclio) | detector | OpenVINO | ✔️ | | diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml new file mode 100644 index 000000000000..d1a19e081abd --- /dev/null +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml @@ -0,0 +1,67 @@ +# Copyright (C) 2023-2024 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +metadata: + name: pth-facebookresearch-sam2-vit-h + namespace: cvat + annotations: + name: Segment Anything 2.0 + version: 2 + type: interactor + spec: + min_pos_points: 1 + min_neg_points: 0 + animated_gif: https://raw.githubusercontent.com/cvat-ai/cvat/develop/site/content/en/images/hrnet_example.gif + help_message: The interactor allows to get a mask of an object using at least one positive, and any negative points inside it + +spec: + description: Interactive object segmentation with Segment-Anything 2.0 + runtime: 'python:3.8' + handler: main:handler + eventTimeout: 30s + + build: + image: cvat.pth.facebookresearch.sam2.vit_h:latest-gpu + baseImage: pytorch/pytorch:2.4.0-cuda12.4-cudnn9-devel + directives: + preCopy: + # set NVIDIA container runtime settings + - kind: ENV + value: NVIDIA_VISIBLE_DEVICES=all + - kind: ENV + value: NVIDIA_DRIVER_CAPABILITIES=compute,utility + # disable interactive frontend + - kind: ENV + value: DEBIAN_FRONTEND=noninteractive + - kind: ENV + value: TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" + # set workdir + - kind: WORKDIR + value: /opt/nuclio/sam2 + # install basic deps + - kind: RUN + value: apt-get update && apt-get -y install build-essential curl git + # install sam2 code + - kind: RUN + value: pip install git+https://github.com/jeanchristopheruel/segment-anything-2.git@main + # download sam2 weights + - kind: RUN + value: curl -O https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: 'http' + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + resources: + limits: + nvidia.com/gpu: 1 + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 + mountMode: volume diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py new file mode 100644 index 000000000000..512c67ed3145 --- /dev/null +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -0,0 +1,42 @@ +# Copyright (C) 2023-2024 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import json +import base64 +from PIL import Image +import io +import torch +from model_handler import ModelHandler + +def init_context(context): + # use bfloat16 for the entire notebook + torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() + + if torch.cuda.get_device_properties(0).major >= 8: + # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + model = ModelHandler() + context.user_data.model = model + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("call handler") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"])) + context.logger.info(f"data: {data}") + image = Image.open(buf) + image = image.convert("RGB") # to make sure image comes in RGB + pos_points = data["pos_points"] + neg_points = data["neg_points"] + + mask = context.user_data.model.handle(image, pos_points, neg_points) + + return context.Response( + body=json.dumps({ 'mask': mask.tolist() }), + headers={}, + content_type='application/json', + status_code=200 + ) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py new file mode 100644 index 000000000000..cf27d006c4e5 --- /dev/null +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py @@ -0,0 +1,28 @@ +# Copyright (C) 2023-2024 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import numpy as np +import torch +from sam2.build_sam import build_sam2 +from sam2.sam2_image_predictor import SAM2ImagePredictor + +class ModelHandler: + def __init__(self): + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + self.sam_checkpoint = "./sam2_hiera_large.pt" + self.model_cfg = "sam2_hiera_l.yaml" + self.predictor = SAM2ImagePredictor(build_sam2(self.model_cfg, self.sam_checkpoint, device="cuda")) + + def handle(self, image, pos_points, neg_points): + pos_points, neg_points = list(pos_points), list(neg_points) + with torch.inference_mode(): + self.predictor.set_image(np.array(image)) + masks, scores, logits = self.predictor.predict( + point_coords=np.array(pos_points + neg_points), + point_labels=np.array([1]*len(pos_points) + [0]*len(neg_points)), + multimask_output=True, + ) + sorted_ind = np.argsort(scores)[::-1] + best_mask = masks[sorted_ind][0] + return best_mask \ No newline at end of file From d482fd9d98c506f19a63b1b37b0af706884620b9 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 00:10:08 -0400 Subject: [PATCH 02/12] Create a changelog fragment --- changelog.d/20240731_000641_ruelj2.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/20240731_000641_ruelj2.md diff --git a/changelog.d/20240731_000641_ruelj2.md b/changelog.d/20240731_000641_ruelj2.md new file mode 100644 index 000000000000..87234e468151 --- /dev/null +++ b/changelog.d/20240731_000641_ruelj2.md @@ -0,0 +1,4 @@ +### Added + +- Added support for the Segment Anything 2.0 as a Nuclio serverless function. The original Facebook Research repository required some modifications (see pull request) to ease the integration with Nuclio. This fork currently fully supports Segment Anything 2.0 on GPU with the large model. + () From 8cff6788603837b96d7f33cc6581d47f565b7010 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 00:34:29 -0400 Subject: [PATCH 03/12] remove useless docstring --- serverless/pytorch/facebookresearch/sam2/nuclio/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py index 512c67ed3145..88cb234dd15f 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -10,9 +10,7 @@ from model_handler import ModelHandler def init_context(context): - # use bfloat16 for the entire notebook torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() - if torch.cuda.get_device_properties(0).major >= 8: # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) torch.backends.cuda.matmul.allow_tf32 = True From ac89a2f884076a5072d66757976a760d032ef558 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 00:56:44 -0400 Subject: [PATCH 04/12] fix readme regarding CPU capabilities for SAM2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d45bdb2254bf..20a7db5fda7d 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ up to 10x. Here is a list of the algorithms we support, and the platforms they c | Name | Type | Framework | CPU | GPU | | ------------------------------------------------------------------------------------------------------- | ---------- | ---------- | --- | --- | -| [Segment Anything 2.0](/serverless/pytorch/facebookresearch/sam2/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | +| [Segment Anything 2.0](/serverless/pytorch/facebookresearch/sam2/nuclio/) | interactor | PyTorch | | ✔️ | | [Segment Anything](/serverless/pytorch/facebookresearch/sam/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | | [Deep Extreme Cut](/serverless/openvino/dextr/nuclio) | interactor | OpenVINO | ✔️ | | | [Faster RCNN](/serverless/openvino/omz/public/faster_rcnn_inception_resnet_v2_atrous_coco/nuclio) | detector | OpenVINO | ✔️ | | From a2ce34f2c7705e441980592461c2774b5d8c9efc Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 02:52:08 -0400 Subject: [PATCH 05/12] reduce the default number of cpu workers to 1 --- .../pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml index d1a19e081abd..eb4298577b01 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml @@ -50,7 +50,7 @@ spec: value: curl -O https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt triggers: myHttpTrigger: - maxWorkers: 2 + maxWorkers: 1 kind: 'http' workerAvailabilityTimeoutMilliseconds: 10000 attributes: From 32a9b2268022af7b713fd08dbfb4e3386afc7477 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 03:32:40 -0400 Subject: [PATCH 06/12] add a try catch block as suggested by coderabbitai --- .../facebookresearch/sam2/nuclio/main.py | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py index 88cb234dd15f..15cd778a7138 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -21,20 +21,29 @@ def init_context(context): context.logger.info("Init context...100%") def handler(context, event): - context.logger.info("call handler") - data = event.body - buf = io.BytesIO(base64.b64decode(data["image"])) - context.logger.info(f"data: {data}") - image = Image.open(buf) - image = image.convert("RGB") # to make sure image comes in RGB - pos_points = data["pos_points"] - neg_points = data["neg_points"] + try: + context.logger.info("call handler") + data = event.body + buf = io.BytesIO(base64.b64decode(data["image"])) + context.logger.info(f"data: {data}") + image = Image.open(buf) + image = image.convert("RGB") # to make sure image comes in RGB + pos_points = data["pos_points"] + neg_points = data["neg_points"] - mask = context.user_data.model.handle(image, pos_points, neg_points) + mask = context.user_data.model.handle(image, pos_points, neg_points) - return context.Response( - body=json.dumps({ 'mask': mask.tolist() }), - headers={}, - content_type='application/json', - status_code=200 - ) + return context.Response( + body=json.dumps({'mask': mask.tolist()}), + headers={}, + content_type='application/json', + status_code=200 + ) + except Exception as e: + context.logger.error(f"Error in handler: {str(e)}") + return context.Response( + body=json.dumps({'error': str(e)}), + headers={}, + content_type='application/json', + status_code=500 + ) From bf650ab0988affc528cc8c550e070e9f066200ab Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Wed, 31 Jul 2024 16:03:10 -0400 Subject: [PATCH 07/12] remove data logs in main --- serverless/pytorch/facebookresearch/sam2/nuclio/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py index 15cd778a7138..b613f650f5a2 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -25,7 +25,6 @@ def handler(context, event): context.logger.info("call handler") data = event.body buf = io.BytesIO(base64.b64decode(data["image"])) - context.logger.info(f"data: {data}") image = Image.open(buf) image = image.convert("RGB") # to make sure image comes in RGB pos_points = data["pos_points"] From 643081fe19b2b2d346378defb13804082fa05cf3 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Fri, 9 Aug 2024 15:53:57 -0400 Subject: [PATCH 08/12] fix: introduce sam2 on cpu --- .../sam2/nuclio/function.yaml | 60 +++++++++++++++++++ .../facebookresearch/sam2/nuclio/main.py | 6 -- .../sam2/nuclio/model_handler.py | 13 +++- 3 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 serverless/pytorch/facebookresearch/sam2/nuclio/function.yaml diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/function.yaml b/serverless/pytorch/facebookresearch/sam2/nuclio/function.yaml new file mode 100644 index 000000000000..20e33ed4a6fe --- /dev/null +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/function.yaml @@ -0,0 +1,60 @@ +# Copyright (C) 2023-2024 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +metadata: + name: pth-facebookresearch-sam2-vit-h + namespace: cvat + annotations: + name: Segment Anything 2.0 + version: 2 + type: interactor + spec: + min_pos_points: 1 + min_neg_points: 0 + animated_gif: https://raw.githubusercontent.com/cvat-ai/cvat/develop/site/content/en/images/hrnet_example.gif + help_message: The interactor allows to get a mask of an object using at least one positive, and any negative points inside it + +spec: + description: Interactive object segmentation with Segment-Anything 2.0 + runtime: 'python:3.8' + handler: main:handler + eventTimeout: 30s + + build: + image: cvat.pth.facebookresearch.sam2.vit_h + baseImage: ubuntu:22.04 + directives: + preCopy: + # disable interactive frontend + - kind: ENV + value: DEBIAN_FRONTEND=noninteractive + # set workdir + - kind: WORKDIR + value: /opt/nuclio/sam2 + # install basic deps + - kind: RUN + value: apt-get update && apt-get -y install build-essential curl git python3 python3-pip ffmpeg libsm6 libxext6 + # install sam2 code + - kind: RUN + value: SAM2_BUILD_CUDA=0 pip install git+https://github.com/facebookresearch/segment-anything-2.git@main + # download sam2 weights + - kind: RUN + value: curl -O https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt + # map pip3 and python3 to pip and python + - kind: RUN + value: ln -s /usr/bin/pip3 /usr/local/bin/pip && ln -s /usr/bin/python3 /usr/bin/python + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: 'http' + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 + mountMode: volume diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py index b613f650f5a2..df2ad42cf0a6 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -10,12 +10,6 @@ from model_handler import ModelHandler def init_context(context): - torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() - if torch.cuda.get_device_properties(0).major >= 8: - # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) - torch.backends.cuda.matmul.allow_tf32 = True - torch.backends.cudnn.allow_tf32 = True - model = ModelHandler() context.user_data.model = model context.logger.info("Init context...100%") diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py index cf27d006c4e5..4bf3afb919fb 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py @@ -9,10 +9,17 @@ class ModelHandler: def __init__(self): - self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + self.device = torch.device('cpu') + if torch.cuda.is_available(): + self.device = torch.device('cuda') + if torch.cuda.get_device_properties(0).major >= 8: + # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + self.sam_checkpoint = "./sam2_hiera_large.pt" self.model_cfg = "sam2_hiera_l.yaml" - self.predictor = SAM2ImagePredictor(build_sam2(self.model_cfg, self.sam_checkpoint, device="cuda")) + self.predictor = SAM2ImagePredictor(build_sam2(self.model_cfg, self.sam_checkpoint, device=self.device)) def handle(self, image, pos_points, neg_points): pos_points, neg_points = list(pos_points), list(neg_points) @@ -25,4 +32,4 @@ def handle(self, image, pos_points, neg_points): ) sorted_ind = np.argsort(scores)[::-1] best_mask = masks[sorted_ind][0] - return best_mask \ No newline at end of file + return best_mask From 6dd2eb709dc08668b354dbed5fd838d08fcdddab Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Fri, 9 Aug 2024 15:54:30 -0400 Subject: [PATCH 09/12] fix: use newest installation mewthod from facebook. (configs files are not embedded into installation package) --- .../pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml index eb4298577b01..d324cf8d4eae 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/function-gpu.yaml @@ -35,7 +35,7 @@ spec: - kind: ENV value: DEBIAN_FRONTEND=noninteractive - kind: ENV - value: TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" + value: TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9+PTX" # set workdir - kind: WORKDIR value: /opt/nuclio/sam2 @@ -44,7 +44,7 @@ spec: value: apt-get update && apt-get -y install build-essential curl git # install sam2 code - kind: RUN - value: pip install git+https://github.com/jeanchristopheruel/segment-anything-2.git@main + value: pip install git+https://github.com/facebookresearch/segment-anything-2.git@main # download sam2 weights - kind: RUN value: curl -O https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt From 288bdcf77a0e02d449c95c96cd6998e5c5dafcaa Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Sat, 10 Aug 2024 01:49:48 -0400 Subject: [PATCH 10/12] update docs -> SAM2 now supports CPU --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 20a7db5fda7d..d45bdb2254bf 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ up to 10x. Here is a list of the algorithms we support, and the platforms they c | Name | Type | Framework | CPU | GPU | | ------------------------------------------------------------------------------------------------------- | ---------- | ---------- | --- | --- | -| [Segment Anything 2.0](/serverless/pytorch/facebookresearch/sam2/nuclio/) | interactor | PyTorch | | ✔️ | +| [Segment Anything 2.0](/serverless/pytorch/facebookresearch/sam2/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | | [Segment Anything](/serverless/pytorch/facebookresearch/sam/nuclio/) | interactor | PyTorch | ✔️ | ✔️ | | [Deep Extreme Cut](/serverless/openvino/dextr/nuclio) | interactor | OpenVINO | ✔️ | | | [Faster RCNN](/serverless/openvino/omz/public/faster_rcnn_inception_resnet_v2_atrous_coco/nuclio) | detector | OpenVINO | ✔️ | | From 158dbbaa9640c1079439554f160790cab9ef5fd4 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Tue, 13 Aug 2024 12:01:07 -0400 Subject: [PATCH 11/12] fix linter errors --- serverless/pytorch/facebookresearch/sam2/nuclio/main.py | 1 - .../pytorch/facebookresearch/sam2/nuclio/model_handler.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py index df2ad42cf0a6..e793a7977543 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/main.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/main.py @@ -6,7 +6,6 @@ import base64 from PIL import Image import io -import torch from model_handler import ModelHandler def init_context(context): diff --git a/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py index 4bf3afb919fb..b34712f257ba 100644 --- a/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py +++ b/serverless/pytorch/facebookresearch/sam2/nuclio/model_handler.py @@ -25,7 +25,7 @@ def handle(self, image, pos_points, neg_points): pos_points, neg_points = list(pos_points), list(neg_points) with torch.inference_mode(): self.predictor.set_image(np.array(image)) - masks, scores, logits = self.predictor.predict( + masks, scores, _ = self.predictor.predict( point_coords=np.array(pos_points + neg_points), point_labels=np.array([1]*len(pos_points) + [0]*len(neg_points)), multimask_output=True, From 618ffbf6bf5c13c430310a32de02f51a22c54b4d Mon Sep 17 00:00:00 2001 From: Jean-Christophe Ruel Date: Thu, 15 Aug 2024 19:29:43 -0400 Subject: [PATCH 12/12] Update 20240731_000641_ruelj2.md --- changelog.d/20240731_000641_ruelj2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/20240731_000641_ruelj2.md b/changelog.d/20240731_000641_ruelj2.md index 87234e468151..f5c65cf0a26e 100644 --- a/changelog.d/20240731_000641_ruelj2.md +++ b/changelog.d/20240731_000641_ruelj2.md @@ -1,4 +1,4 @@ ### Added -- Added support for the Segment Anything 2.0 as a Nuclio serverless function. The original Facebook Research repository required some modifications (see pull request) to ease the integration with Nuclio. This fork currently fully supports Segment Anything 2.0 on GPU with the large model. +- Added support for the Segment Anything 2.0 as a Nuclio serverless function. Currently fully supports SAM2 on GPU and CPU. ()