Merge branch 'main' into feat-add-cloud-run-gemma2-rag-sample

.github/actions/spelling/line_forbidden.patterns

@@ -260,8 +260,9 @@
 \bVertex\s?Search\b
 
 # Should be Colab
-\scolab\b
-\b[Cc]ollab\b
+\scolabs?\b
+\b[Cc]ollabs?\b
+\b[Cc]oLabs?\b
 
 # Should be Kaggle
 \skaggle\b

CONTRIBUTING.md

@@ -90,3 +90,17 @@ Here are some additional things to keep in mind during the process:
 - **Read the [Google's Open Source Community Guidelines](https://opensource.google/conduct/).** The contribution guidelines will provide you with more information about the project and how to contribute.
 - **Test your changes.** Before you submit a pull request, make sure that your changes work as expected.
 - **Be patient.** It may take some time for your pull request to be reviewed and merged.
+
+---
+
+## For Google Employees
+
+Complete the following steps to register your GitHub account and be added as a contributor to this repository.
+
+1. Register your GitHub account at [go/github](http://go/github)
+
+1. Once you have registered, go to [go/github-googlecloudplatform](http://go/github-googlecloudplatform) and request to join the GoogleCloudPlatform organization. Check the box "I need write access on a public repository".
+
+1. You'll receive an email to your GitHub registered email to approve the request to join. Approve it.
+
+1. Request to join this team [GoogleCloudPlatform/teams/generative-ai-samples-contributors](https://github.com/orgs/GoogleCloudPlatform/teams/generative-ai-samples-contributors/members)

gemini/evaluation/evaluate_gemini_with_autosxs.ipynb

@@ -422,7 +422,6 @@
    },
    "outputs": [],
    "source": [
-    "import os\n",
     "import pprint\n",
     "\n",
     "import pandas as pd\n",
@@ -665,7 +664,7 @@
     "job = aiplatform.PipelineJob(\n",
     "    job_id=display_name,\n",
     "    display_name=display_name,\n",
-    "    pipeline_root=os.path.join(BUCKET_URI, display_name),\n",
+    "    pipeline_root=f\"{BUCKET_URI}/{display_name}\",\n",
     "    template_path=TEMPLATE_URI,\n",
     "    parameter_values=parameters,\n",
     "    enable_caching=False,\n",
@@ -855,7 +854,7 @@
     "job = aiplatform.PipelineJob(\n",
     "    job_id=display_name,\n",
     "    display_name=display_name,\n",
-    "    pipeline_root=os.path.join(BUCKET_URI, display_name),\n",
+    "    pipeline_root=f\"{BUCKET_URI}/{display_name}\",\n",
     "    template_path=TEMPLATE_URI,\n",
     "    parameter_values=parameters,\n",
     "    enable_caching=False,\n",

gemini/use-cases/retail/product_attributes_extraction.ipynb

@@ -63,7 +63,7 @@
    "source": [
     "| | |\n",
     "|-|-|\n",
-    "|Author(s) | [Tianli Yu](https://github.com/tianli |"
+    "|Author(s) | [Tianli Yu](https://github.com/tianli) |"
    ]
   },
   {
@@ -129,22 +129,18 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {
-    "id": "XRvKdaPDTznN",
-    "outputId": "01b6924b-740d-43d8-8a07-be8ecdad403e",
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    }
+    "id": "XRvKdaPDTznN"
    },
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "{'status': 'ok', 'restart': True}"
       ]
      },
+     "execution_count": 1,
      "metadata": {},
-     "execution_count": 1
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -242,6 +238,8 @@
    "source": [
     "# @title Image loading and parsing library\n",
     "import json\n",
+    "from typing import Union\n",
+    "from vertexai.preview.generative_models import Part\n",
     "\n",
     "import requests\n",
     "\n",
@@ -257,7 +255,33 @@
     "        return \"image/jpeg\"\n",
     "\n",
     "\n",
-    "def parse_json_from_markdown(answer: str) -> dict[str, list[str]]:\n",
+    "def load_image_part_from_uri(image_uri: str) -> Union[None, Part]:\n",
+    "    \"\"\"Load image to a prompt Part from a remote or local file URI.\n",
+    "\n",
+    "    Args:\n",
+    "        image_uri (str): The uri or the local file path for the image.\n",
+    "\n",
+    "    Returns:\n",
+    "        The prompt Part object.\n",
+    "    \"\"\"\n",
+    "    if image_uri.startswith(\"gs://\"):\n",
+    "        return Part.from_uri(image_uri, mime_type=get_mime_from_uri(image_uri))\n",
+    "    elif image_uri.startswith(\"http://\") or image_uri.startswith(\"https://\"):\n",
+    "        response = requests.get(image_uri)\n",
+    "        if response.status_code == 200:\n",
+    "            image_bytes = response.content\n",
+    "            return Part.from_data(image_bytes, mime_type=get_mime_from_uri(image_uri))\n",
+    "        else:\n",
+    "            print(\n",
+    "                f\"Fetch image failed for {image_uri}, status code: {response.status_code}\"\n",
+    "            )\n",
+    "            return None\n",
+    "    else:\n",
+    "        image_bytes = open(image_uri, \"rb\").read()\n",
+    "        return Part.from_data(image_bytes, mime_type=get_mime_from_uri(image_uri))\n",
+    "\n",
+    "\n",
+    "def parse_json_from_markdown(answer: str) -> str:\n",
     "    \"\"\"Parse the json from the markdown answer.\n",
     "\n",
     "    Args:\n",
@@ -279,7 +303,7 @@
     "        if answer.startswith(\"json\"):\n",
     "            answer = answer[4:]\n",
     "        result = json.loads(answer)\n",
-    "    return result"
+    "    return json.dumps(result)"
    ]
   },
   {
@@ -291,7 +315,10 @@
    "outputs": [],
    "source": [
     "# @title The ProductImageAgent Class.\n",
+    "from typing import Union\n",
+    "\n",
     "import ipywidgets as widgets\n",
+    "from IPython.display import display\n",
     "from vertexai.preview.generative_models import GenerationConfig, GenerativeModel, Part\n",
     "\n",
     "\n",
@@ -301,15 +328,15 @@
     "\n",
     "    Args:\n",
     "        gemini_model_version (str): The version string of the Gemini 1.5 model.\n",
-    "            gemini-1.5-pro or gemini-1.5-flash\n",
+    "            gemini-1.5-pro-001 or gemini-1.5-flash-001\n",
     "        temperature (float): The temperature of the model. Defaults to 1.0.\n",
     "        max_output_tokens (int): The maximum number of output tokens. Defaults to\n",
     "            8192.\n",
     "    \"\"\"\n",
     "\n",
     "    def __init__(\n",
     "        self,\n",
-    "        gemini_model_version: str = \"gemini-1.5-pro\",\n",
+    "        gemini_model_version: str = \"gemini-1.5-pro-001\",\n",
     "        temperature: float = 0.0,\n",
     "        max_output_tokens: int = 8192,\n",
     "    ):\n",
@@ -340,7 +367,9 @@
     "        Returns:\n",
     "            The generated detailed description from the model response.\n",
     "        \"\"\"\n",
-    "        image_part = Part.from_uri(image_uri, mime_type=get_mime_from_uri(image_uri))\n",
+    "        image_part = load_image_part_from_uri(image_uri)\n",
+    "        if image_part is None:\n",
+    "            return \"\"\n",
     "        prompt = \"\"\"\n",
     "        Please write a complete and detailed product description for the\n",
     "        above product image. The length of the description should be at least\n",
@@ -359,8 +388,8 @@
     "    def get_attributes(\n",
     "        self,\n",
     "        image_uri: str,\n",
-    "        product_category: str = None,\n",
-    "        vocabulary_json: str = None,\n",
+    "        product_category: Union[None, str] = None,\n",
+    "        vocabulary_json: Union[None, str] = None,\n",
     "        debug: bool = False,\n",
     "    ) -> str:\n",
     "        \"\"\"Generates the product attributes from an image.\n",
@@ -373,7 +402,9 @@
     "        Returns:\n",
     "            The product attribute json string from the model response.\n",
     "        \"\"\"\n",
-    "        image_part = Part.from_uri(image_uri, mime_type=get_mime_from_uri(image_uri))\n",
+    "        image_part = load_image_part_from_uri(image_uri)\n",
+    "        if image_part is None:\n",
+    "            return \"\"\n",
     "        if product_category:\n",
     "            prompt = f\"\"\"\n",
     "          The above image is a product image from the {product_category}\n",
@@ -409,8 +440,8 @@
     "    def get_attributes_self_correcting_prompt(\n",
     "        self,\n",
     "        image_uri: str,\n",
-    "        product_category: str = None,\n",
-    "        vocabulary_json: str = None,\n",
+    "        product_category: Union[None, str] = None,\n",
+    "        vocabulary_json: Union[None, str] = None,\n",
     "        debug: bool = False,\n",
     "    ) -> str:\n",
     "        \"\"\"Generates the product attributes from an image using self-correcting prompt.\n",
@@ -423,7 +454,9 @@
     "        Returns:\n",
     "            The product attribute json string from the parsed model response.\n",
     "        \"\"\"\n",
-    "        image_part = Part.from_uri(image_uri, mime_type=get_mime_from_uri(image_uri))\n",
+    "        image_part = load_image_part_from_uri(image_uri)\n",
+    "        if image_part is None:\n",
+    "            return \"\"\n",
     "        if product_category:\n",
     "            prompt = f\"\"\"\n",
     "          The above image is a product image from the {product_category}\n",
@@ -489,7 +522,7 @@
    "outputs": [],
    "source": [
     "# @title Example 1: Generate detailed product description from an image.\n",
-    "image_uri = \"https://plus.unsplash.com/premium_photo-1711051513016-72baa1035293?q=80&w=3687&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\"  # @param {type:\"string\"}\n",
+    "image_uri = \"IMAGE_URI\"  # @param {type:\"string\"}\n",
     "\n",
     "product_agent.get_detailed_description(image_uri, True)"
    ]
@@ -503,7 +536,7 @@
    "outputs": [],
    "source": [
     "# @title Example 2: Get product attribute json from an image -- open vocabulary.\n",
-    "image_uri = \"https://plus.unsplash.com/premium_photo-1711051513016-72baa1035293?q=80&w=3687&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\"  # @param {type:\"string\"}\n",
+    "image_uri = \"IMAGE_URI\"  # @param {type:\"string\"}\n",
     "\n",
     "# Open vocabulary.\n",
     "attribute_json = product_agent.get_attributes(image_uri, debug=True)\n",
@@ -519,7 +552,7 @@
    "outputs": [],
    "source": [
     "# @title Example 3: Get product attribute json from an image -- closed vocabulary.\n",
-    "image_uri = \"https://plus.unsplash.com/premium_photo-1711051513016-72baa1035293?q=80&w=3687&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\"  # @param {type:\"string\"}\n",
+    "image_uri = \"IMAGE_URI\"  # @param {type:\"string\"}\n",
     "\n",
     "# Closed vocabulary.\n",
     "vocabulary = \"\"\"\n",
@@ -541,7 +574,7 @@
    "outputs": [],
    "source": [
     "# @title Example 4: Get product attribute json from an image using self-correcting prompt -- open vocabulary\n",
-    "image_uri = \"https://plus.unsplash.com/premium_photo-1711051513016-72baa1035293?q=80&w=3687&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\"  # @param {type:\"string\"}\n",
+    "image_uri = \"IMAGE_URI\"  # @param {type:\"string\"}\n",
     "\n",
     "# Open vocabulary.\n",
     "attribute_json = product_agent.get_attributes_self_correcting_prompt(\n",
@@ -559,7 +592,7 @@
    "outputs": [],
    "source": [
     "# @title Example 5: Get product attribute json from an image using self-correcting prompt -- closed vocabulary\n",
-    "image_uri = \"https://plus.unsplash.com/premium_photo-1711051513016-72baa1035293?q=80&w=3687&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D\"  # @param {type:\"string\"}\n",
+    "image_uri = \"IMAGE_URI\"  # @param {type:\"string\"}\n",
     "\n",
     "# Closed vocabulary.\n",
     "vocabulary = \"\"\"\n",
@@ -575,8 +608,7 @@
  "metadata": {
   "colab": {
    "name": "product_attributes_extraction.ipynb",
-   "toc_visible": true,
-   "provenance": []
+   "toc_visible": true
   },
   "kernelspec": {
    "display_name": "Python 3",