feat: fast enum cache updates

.github/workflows/examples.yml

@@ -87,6 +87,7 @@ jobs:
           PDL_API_KEY: ${{secrets.PDL_API_KEY}}
           COMPOSIO_API_KEY: ${{ inputs.api_key || secrets.COMPOSIO_API_KEY_PROD }}
           COMPOSIO_BASE_URL: ${{ inputs.base_url || secrets.COMPOSIO_BASE_URL || 'https://backend.composio.dev/api' }}
+          COMPOSIO_NO_CACHE_REFRESH: "true"
           # TODO(@kaavee): Add Anthropic API key
         run: |
           python -m pip install uv

python/composio/cli/apps.py

@@ -7,11 +7,9 @@
 
 import ast
 import os.path
-import shutil
 
 import click
 
-from composio import constants
 from composio.cli.context import Context, pass_context
 from composio.cli.utils.decorators import handle_exceptions
 from composio.cli.utils.helpfulcmd import HelpfulCmdBase
@@ -80,12 +78,17 @@ def _update(context: Context) -> None:
 
 @_apps.command(name="generate-types")
 @click.help_option("--help", "-h", "-help")
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Forcefully update the type stubs.",
+)
 @handle_exceptions()
 @pass_context
-def _generate_types(context: Context) -> None:
+def _generate_types(context: Context, force: bool = False) -> None:
     """Updates the local type stubs with the latest app data."""
     context.console.print("Fetching latest data from Composio API...")
-    generate_type_stubs(context.client)
+    generate_type_stubs(context.client, force=force)
     context.console.print(
         "[green]Successfully updated type stubs for Apps, Actions, and Triggers[/green]"
     )
@@ -134,16 +137,10 @@ def generate_type_stub(enum_file: str, cache_folder: os.PathLike) -> None:
         f.write(ast.unparse(tree))
 
 
-def generate_type_stubs(client: Composio) -> None:
-    # Update local cache first
-    for cache_folder in ["apps", "actions", "triggers", "tags"]:
-        shutil.rmtree(
-            constants.LOCAL_CACHE_DIRECTORY / cache_folder,
-            ignore_errors=True,
-        )
-    apps = update_apps(client)
-    update_actions(client, apps)
-    update_triggers(client, apps)
+def generate_type_stubs(client: Composio, force: bool = False) -> None:
+    update_apps(client=client, force=force)
+    update_actions(client=client, force=force)
+    update_triggers(client=client, force=force)
 
     enums_folder = os.path.join(os.path.dirname(__file__), "..", "client", "enums")
     apps_enum = os.path.join(enums_folder, "app.py")

python/composio/client/__init__.py

@@ -67,6 +67,7 @@ def __init__(
         api_key: t.Optional[str] = None,
         base_url: t.Optional[str] = None,
         runtime: t.Optional[str] = None,
+        check_cache: bool = True,
     ) -> None:
         """
         Initialize Composio SDK client
@@ -88,6 +89,13 @@ def __init__(
         self.logs = Logs(client=self)
         _clients.append(self)
 
+        if check_cache:
+            from composio.client.utils import (  # pylint: disable=import-outside-toplevel
+                check_cache_refresh,
+            )
+
+            check_cache_refresh(self)
+
     @staticmethod
     def get_latest() -> "Composio":
         """Get latest composio client from the runtime stack."""

python/composio/client/base.py

@@ -45,17 +45,18 @@ def _raise_if_required(
                 not match with the expected status code
         """
         if response.status_code != status_code:
-            raise HTTPError(
-                message=response.content.decode(encoding="utf-8"),
-                status_code=response.status_code,
-            )
+            raise HTTPError(message=response.text, status_code=response.status_code)
         return response
 
     def get(self, queries: t.Optional[t.Dict[str, str]] = None) -> t.List[ModelType]:
         """List available models."""
         request = self._raise_if_required(
             response=self.client.http.get(
-                url=str(self.endpoint(queries=queries or {})),
+                url=str(
+                    self.endpoint(
+                        queries=queries or {},
+                    )
+                ),
             ),
         )
 

python/composio/client/collections.py

@@ -358,6 +358,22 @@ def get(self, name: t.Optional[str] = None) -> t.Union[AppModel, t.List[AppModel
 
         return super().get(queries={})
 
+    def list_enums(self) -> list[str]:
+        """Get just the app names on the server."""
+        response = self._raise_if_required(
+            response=self.client.http.get(
+                str(self.endpoint / "list" / "enums"),
+            )
+        )
+        return response.text.split("\n")
+
+    def get_etag(self) -> str:
+        return self._raise_if_required(
+            response=self.client.http.head(
+                str(self.endpoint / "list" / "enums"),
+            )
+        ).headers["ETag"]
+
 
 class TypeModel(BaseModel):
     type: str
@@ -958,6 +974,13 @@ def subscribe(self, timeout: float = 15.0) -> TriggerSubscription:
             timeout=timeout,
         )
 
+    def get_etag(self) -> str:
+        return self._raise_if_required(
+            response=self.client.http.head(
+                str(self.endpoint / "list" / "enums"),
+            )
+        ).headers["ETag"]
+
 
 class ActiveTriggerModel(BaseModel):
     """Active trigger data model."""
@@ -1027,6 +1050,7 @@ class ActionModel(BaseModel):
     appId: str
     version: str
     available_versions: t.List[str]
+    no_auth: bool
 
     tags: t.List[str]
     logo: t.Optional[str] = None
@@ -1097,9 +1121,9 @@ def _get_action(self, action: ActionType) -> ActionModel:
 
     def _get_actions(
         self,
-        actions: t.Optional[t.Sequence[ActionType]] = None,
-        apps: t.Optional[t.Sequence[AppType]] = None,
-        tags: t.Optional[t.Sequence[TagType]] = None,
+        actions: t.Optional[t.Collection[ActionType]] = None,
+        apps: t.Optional[t.Collection[AppType]] = None,
+        tags: t.Optional[t.Collection[TagType]] = None,
         limit: t.Optional[int] = None,
         use_case: t.Optional[str] = None,
         allow_all: bool = False,
@@ -1416,6 +1440,21 @@ def search_for_a_task(
             for task in response.json().get("items", [])
         ]
 
+    def list_enums(self) -> list[str]:
+        """Get just the action names on the server"""
+        return self._raise_if_required(
+            response=self.client.http.get(
+                str(self.endpoint / "list" / "enums"),
+            )
+        ).text.split("\n")
+
+    def get_etag(self) -> str:
+        return self._raise_if_required(
+            response=self.client.http.head(
+                str(self.endpoint / "list" / "enums"),
+            )
+        ).headers["ETag"]
+
     def create_file_upload(
         self,
         app: str,

python/composio/client/enums/action.py

@@ -1,7 +1,7 @@
 import typing as t
 import warnings
 
-from composio.client.enums.base import ActionData, replacement_action_name
+from composio.client.enums.base import ActionData, create_action
 from composio.client.enums.enum import Enum, EnumGenerator
 from composio.constants import VERSION_LATEST, VERSION_LATEST_BASE
 from composio.exceptions import EnumMetadataNotFound, InvalidVersionString, VersionError
@@ -86,24 +86,7 @@ def fetch_and_cache(self) -> t.Optional[ActionData]:
         if "appName" not in response:
             return None
 
-        replaced_by = replacement_action_name(
-            response["description"], response["appName"]
-        )
-        return ActionData(  # type: ignore
-            name=response["name"],
-            app=response["appName"],
-            tags=response["tags"],
-            no_auth=(
-                client.http.get(url=str(client.apps.endpoint / response["appName"]))
-                .json()
-                .get("no_auth", False)
-            ),
-            is_local=False,
-            is_runtime=False,
-            shell=False,
-            path=self.storage_path,
-            replaced_by=replaced_by,
-        )
+        return create_action(response, self.storage_path)
 
     @property
     def name(self) -> str:

python/composio/client/enums/action.pyi

@@ -1,7 +1,7 @@
 import typing as t
 import warnings
 
-from composio.client.enums.base import ActionData, replacement_action_name
+from composio.client.enums.base import ActionData, create_action
 from composio.client.enums.enum import Enum, EnumGenerator
 from composio.constants import VERSION_LATEST, VERSION_LATEST_BASE
 from composio.exceptions import EnumMetadataNotFound, InvalidVersionString, VersionError
@@ -3125,6 +3125,12 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     FIREFLIES_GET_USERS: "Action"
     FIREFLIES_GET_USER_BY_ID: "Action"
     FIREFLIES_UPLOAD_AUDIO: "Action"
+    FOURSQAURE_V2_PLACE_SEARCH: "Action"
+    FOURSQUARE_V1_FIND_NEARBY_PLACES: "Action"
+    FOURSQUARE_V1_GET_PLACE_DETAILS: "Action"
+    FOURSQUARE_V1_GET_PLACE_PHOTOS: "Action"
+    FOURSQUARE_V1_GET_PLACE_TIPS: "Action"
+    FOURSQUARE_V1_PLACE_SEARCH: "Action"
     FRESHDESK_CREATE_TICKET: "Action"
     FRESHDESK_DELETE_TICKET: "Action"
     FRESHDESK_GET_TICKETS: "Action"
@@ -4088,6 +4094,21 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     GIT_GITHUB_CLONE_CMD: "Action"
     GIT_GIT_REPO_TREE: "Action"
     GMAIL_ADD_LABEL_TO_EMAIL: "Action"
+    GMAIL_BETA_ADD_LABEL_TO_EMAIL: "Action"
+    GMAIL_BETA_CREATE_EMAIL_DRAFT: "Action"
+    GMAIL_BETA_CREATE_LABEL: "Action"
+    GMAIL_BETA_FETCH_EMAILS: "Action"
+    GMAIL_BETA_FETCH_MESSAGE_BY_MESSAGE_ID: "Action"
+    GMAIL_BETA_FETCH_MESSAGE_BY_THREAD_ID: "Action"
+    GMAIL_BETA_GET_ATTACHMENT: "Action"
+    GMAIL_BETA_GET_PEOPLE: "Action"
+    GMAIL_BETA_GET_PROFILE: "Action"
+    GMAIL_BETA_LIST_LABELS: "Action"
+    GMAIL_BETA_LIST_THREADS: "Action"
+    GMAIL_BETA_MODIFY_THREAD_LABELS: "Action"
+    GMAIL_BETA_REMOVE_LABEL: "Action"
+    GMAIL_BETA_REPLY_TO_THREAD: "Action"
+    GMAIL_BETA_SEND_EMAIL: "Action"
     GMAIL_CREATE_EMAIL_DRAFT: "Action"
     GMAIL_CREATE_LABEL: "Action"
     GMAIL_FETCH_EMAILS: "Action"
@@ -4168,6 +4189,16 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     GOOGLETASKS_LIST_TASK_LISTS: "Action"
     GOOGLETASKS_PATCH_TASK: "Action"
     GOOGLETASKS_PATCH_TASK_LIST: "Action"
+    GOOGLE_DRIVE_BETA_ADD_FILE_SHARING_PREFERENCE: "Action"
+    GOOGLE_DRIVE_BETA_COPY_FILE: "Action"
+    GOOGLE_DRIVE_BETA_CREATE_FILE_FROM_TEXT: "Action"
+    GOOGLE_DRIVE_BETA_CREATE_FOLDER: "Action"
+    GOOGLE_DRIVE_BETA_DELETE_FOLDER_OR_FILE: "Action"
+    GOOGLE_DRIVE_BETA_EDIT_FILE: "Action"
+    GOOGLE_DRIVE_BETA_FIND_FILE: "Action"
+    GOOGLE_DRIVE_BETA_FIND_FOLDER: "Action"
+    GOOGLE_DRIVE_BETA_PARSE_FILE: "Action"
+    GOOGLE_DRIVE_BETA_UPLOAD_FILE: "Action"
     GOOGLE_MAPS_GET_DIRECTION: "Action"
     GOOGLE_MAPS_GET_ROUTE: "Action"
     GOOGLE_MAPS_NEARBY_SEARCH: "Action"
@@ -4444,11 +4475,6 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     HUBSPOT_UPDATE_TOKEN_ON_EVENT_TEMPLATE: "Action"
     HUBSPOT_UPDATE_VIDEO_CONFERENCE_APP_SETTINGS: "Action"
     IMAGE_ANALYSER_ANALYSE: "Action"
-    INDUCED_AI_EXTRACT_DATA: "Action"
-    INDUCED_AI_GET_AUTONOMOUS_TASK_STATUS: "Action"
-    INDUCED_AI_GET_DATA_EXTRACTION_STATUS: "Action"
-    INDUCED_AI_PERFORM_AUTONOMOUS_TASK: "Action"
-    INDUCED_AI_STOP_AUTONOMOUS_TASK: "Action"
     INTERCOM_ADD_SUBSCRIPTION_TO_A_CONTACT: "Action"
     INTERCOM_ADD_TAG_TO_A_CONTACT: "Action"
     INTERCOM_ATTACH_A_CONTACT_TO_A_COMPANY: "Action"
@@ -5267,6 +5293,8 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     LINEAR_LIST_LINEAR_LABELS: "Action"
     LINEAR_LIST_LINEAR_PROJECTS: "Action"
     LINEAR_LIST_LINEAR_TEAMS: "Action"
+    LINKEDIN_BETA_GET_RELATIONS: "Action"
+    LINKEDIN_BETA_SEARCH: "Action"
     LINKEDIN_CREATE_LINKED_IN_POST: "Action"
     LINKEDIN_DELETE_LINKED_IN_POST: "Action"
     LINKEDIN_GET_COMPANY_INFO: "Action"
@@ -5763,6 +5791,7 @@ class Action(Enum[ActionData], metaclass=EnumGenerator):
     NOTION_UPDATE_SCHEMA_DATABASE: "Action"
     ONEPAGE_SEARCH_INPUT_POST_REQUEST: "Action"
     ONEPAGE_TOKEN_DETAILS_REQUEST: "Action"
+    ONE_DRIVE_DOWNLOAD_FILE: "Action"
     ONE_DRIVE_ONEDRIVE_CREATE_FOLDER: "Action"
     ONE_DRIVE_ONEDRIVE_CREATE_TEXT_FILE: "Action"
     ONE_DRIVE_ONEDRIVE_FIND_FILE: "Action"

python/composio/client/enums/app.pyi

@@ -126,6 +126,8 @@ class App(Enum[AppData], metaclass=EnumGenerator):
     FOMO: "App"
     FORMCARRY: "App"
     FORMSITE: "App"
+    FOURSQAURE_V2: "App"
+    FOURSQUARE_V1: "App"
     FRESHBOOKS: "App"
     FRESHDESK: "App"
     FRONT: "App"
@@ -159,7 +161,6 @@ class App(Enum[AppData], metaclass=EnumGenerator):
     ICIMS_TALENT_CLOUD: "App"
     IDEA_SCALE: "App"
     IMAGE_ANALYSER: "App"
-    INDUCED_AI: "App"
     INTERCOM: "App"
     INTERZOID: "App"
     JIRA: "App"

python/composio/client/enums/base.py

@@ -3,6 +3,7 @@
 """
 
 import typing as t
+from pathlib import Path
 
 from pydantic import Field
 
@@ -24,6 +25,12 @@
 ACTIONS_CACHE = LOCAL_CACHE_DIRECTORY / "actions"
 TRIGGERS_CACHE = LOCAL_CACHE_DIRECTORY / "triggers"
 
+APPS_ETAG = LOCAL_CACHE_DIRECTORY / "apps.etag"
+ACTIONS_ETAG = LOCAL_CACHE_DIRECTORY / "actions.etag"
+TRIGGERS_ETAG = LOCAL_CACHE_DIRECTORY / "triggers.etag"
+
+DEPRECATED_MARKER = "<<DEPRECATED use "
+
 
 class SentinalObject:
     """Sentinel object."""
@@ -111,13 +118,26 @@ def get_runtime_actions() -> t.List:
     return list(_runtime_actions)
 
 
-DEPRECATED_MARKER = "<<DEPRECATED use "
-
-
 def replacement_action_name(description: str, app_name: str) -> t.Optional[str]:
     """If the action is deprecated, get the replacement action name."""
     if description is not None and DEPRECATED_MARKER in description:
         _, newact = description.split(DEPRECATED_MARKER, maxsplit=1)
         return (app_name + "_" + newact.replace(">>", "")).upper()
 
     return None
+
+
+def create_action(response: dict[str, t.Any], storage_path: Path) -> ActionData:
+    return ActionData(
+        name=response["name"],
+        app=response["appName"],
+        tags=response["tags"],
+        no_auth=response["no_auth"],
+        is_local=False,
+        is_runtime=False,
+        shell=False,
+        path=storage_path,
+        replaced_by=replacement_action_name(
+            response["description"], response["appName"]
+        ),
+    )