diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index ff745570d73..c20b19895db 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -87,6 +87,7 @@ jobs: PDL_API_KEY: ${{secrets.PDL_API_KEY}} COMPOSIO_API_KEY: ${{ inputs.api_key || secrets.COMPOSIO_API_KEY_PROD }} COMPOSIO_BASE_URL: ${{ inputs.base_url || secrets.COMPOSIO_BASE_URL || 'https://backend.composio.dev/api' }} + COMPOSIO_NO_CACHE_REFRESH: "true" # TODO(@kaavee): Add Anthropic API key run: | python -m pip install uv diff --git a/python/composio/cli/apps.py b/python/composio/cli/apps.py index e580540442b..c913c4d08d3 100755 --- a/python/composio/cli/apps.py +++ b/python/composio/cli/apps.py @@ -7,11 +7,9 @@ import ast import os.path -import shutil import click -from composio import constants from composio.cli.context import Context, pass_context from composio.cli.utils.decorators import handle_exceptions from composio.cli.utils.helpfulcmd import HelpfulCmdBase @@ -80,12 +78,17 @@ def _update(context: Context) -> None: @_apps.command(name="generate-types") @click.help_option("--help", "-h", "-help") +@click.option( + "--force", + is_flag=True, + help="Forcefully update the type stubs.", +) @handle_exceptions() @pass_context -def _generate_types(context: Context) -> None: +def _generate_types(context: Context, force: bool = False) -> None: """Updates the local type stubs with the latest app data.""" context.console.print("Fetching latest data from Composio API...") - generate_type_stubs(context.client) + generate_type_stubs(context.client, force=force) context.console.print( "[green]Successfully updated type stubs for Apps, Actions, and Triggers[/green]" ) @@ -134,16 +137,10 @@ def generate_type_stub(enum_file: str, cache_folder: os.PathLike) -> None: f.write(ast.unparse(tree)) -def generate_type_stubs(client: Composio) -> None: - # Update local cache first - for cache_folder in ["apps", "actions", "triggers", "tags"]: - shutil.rmtree( - constants.LOCAL_CACHE_DIRECTORY / cache_folder, - ignore_errors=True, - ) - apps = update_apps(client) - update_actions(client, apps) - update_triggers(client, apps) +def generate_type_stubs(client: Composio, force: bool = False) -> None: + update_apps(client=client, force=force) + update_actions(client=client, force=force) + update_triggers(client=client, force=force) enums_folder = os.path.join(os.path.dirname(__file__), "..", "client", "enums") apps_enum = os.path.join(enums_folder, "app.py") diff --git a/python/composio/client/__init__.py b/python/composio/client/__init__.py index 0842c930b91..d6abb1ba87f 100644 --- a/python/composio/client/__init__.py +++ b/python/composio/client/__init__.py @@ -67,6 +67,7 @@ def __init__( api_key: t.Optional[str] = None, base_url: t.Optional[str] = None, runtime: t.Optional[str] = None, + check_cache: bool = True, ) -> None: """ Initialize Composio SDK client @@ -88,6 +89,13 @@ def __init__( self.logs = Logs(client=self) _clients.append(self) + if check_cache: + from composio.client.utils import ( # pylint: disable=import-outside-toplevel + check_cache_refresh, + ) + + check_cache_refresh(self) + @staticmethod def get_latest() -> "Composio": """Get latest composio client from the runtime stack.""" diff --git a/python/composio/client/base.py b/python/composio/client/base.py index b79637f5234..03736392526 100644 --- a/python/composio/client/base.py +++ b/python/composio/client/base.py @@ -45,17 +45,18 @@ def _raise_if_required( not match with the expected status code """ if response.status_code != status_code: - raise HTTPError( - message=response.content.decode(encoding="utf-8"), - status_code=response.status_code, - ) + raise HTTPError(message=response.text, status_code=response.status_code) return response def get(self, queries: t.Optional[t.Dict[str, str]] = None) -> t.List[ModelType]: """List available models.""" request = self._raise_if_required( response=self.client.http.get( - url=str(self.endpoint(queries=queries or {})), + url=str( + self.endpoint( + queries=queries or {}, + ) + ), ), ) diff --git a/python/composio/client/collections.py b/python/composio/client/collections.py index 9f891e34012..b271e82b6d6 100644 --- a/python/composio/client/collections.py +++ b/python/composio/client/collections.py @@ -358,6 +358,22 @@ def get(self, name: t.Optional[str] = None) -> t.Union[AppModel, t.List[AppModel return super().get(queries={}) + def list_enums(self) -> list[str]: + """Get just the app names on the server.""" + response = self._raise_if_required( + response=self.client.http.get( + str(self.endpoint / "list" / "enums"), + ) + ) + return response.text.split("\n") + + def get_etag(self) -> str: + return self._raise_if_required( + response=self.client.http.head( + str(self.endpoint / "list" / "enums"), + ) + ).headers["ETag"] + class TypeModel(BaseModel): type: str @@ -958,6 +974,13 @@ def subscribe(self, timeout: float = 15.0) -> TriggerSubscription: timeout=timeout, ) + def get_etag(self) -> str: + return self._raise_if_required( + response=self.client.http.head( + str(self.endpoint / "list" / "enums"), + ) + ).headers["ETag"] + class ActiveTriggerModel(BaseModel): """Active trigger data model.""" @@ -1027,6 +1050,7 @@ class ActionModel(BaseModel): appId: str version: str available_versions: t.List[str] + no_auth: bool tags: t.List[str] logo: t.Optional[str] = None @@ -1097,9 +1121,9 @@ def _get_action(self, action: ActionType) -> ActionModel: def _get_actions( self, - actions: t.Optional[t.Sequence[ActionType]] = None, - apps: t.Optional[t.Sequence[AppType]] = None, - tags: t.Optional[t.Sequence[TagType]] = None, + actions: t.Optional[t.Collection[ActionType]] = None, + apps: t.Optional[t.Collection[AppType]] = None, + tags: t.Optional[t.Collection[TagType]] = None, limit: t.Optional[int] = None, use_case: t.Optional[str] = None, allow_all: bool = False, @@ -1416,6 +1440,21 @@ def search_for_a_task( for task in response.json().get("items", []) ] + def list_enums(self) -> list[str]: + """Get just the action names on the server""" + return self._raise_if_required( + response=self.client.http.get( + str(self.endpoint / "list" / "enums"), + ) + ).text.split("\n") + + def get_etag(self) -> str: + return self._raise_if_required( + response=self.client.http.head( + str(self.endpoint / "list" / "enums"), + ) + ).headers["ETag"] + def create_file_upload( self, app: str, diff --git a/python/composio/client/enums/action.py b/python/composio/client/enums/action.py index 4a586b6f675..686c5895596 100644 --- a/python/composio/client/enums/action.py +++ b/python/composio/client/enums/action.py @@ -1,7 +1,7 @@ import typing as t import warnings -from composio.client.enums.base import ActionData, replacement_action_name +from composio.client.enums.base import ActionData, create_action from composio.client.enums.enum import Enum, EnumGenerator from composio.constants import VERSION_LATEST, VERSION_LATEST_BASE from composio.exceptions import EnumMetadataNotFound, InvalidVersionString, VersionError @@ -86,24 +86,7 @@ def fetch_and_cache(self) -> t.Optional[ActionData]: if "appName" not in response: return None - replaced_by = replacement_action_name( - response["description"], response["appName"] - ) - return ActionData( # type: ignore - name=response["name"], - app=response["appName"], - tags=response["tags"], - no_auth=( - client.http.get(url=str(client.apps.endpoint / response["appName"])) - .json() - .get("no_auth", False) - ), - is_local=False, - is_runtime=False, - shell=False, - path=self.storage_path, - replaced_by=replaced_by, - ) + return create_action(response, self.storage_path) @property def name(self) -> str: diff --git a/python/composio/client/enums/action.pyi b/python/composio/client/enums/action.pyi index 9cc0f6cd155..8778c035ac7 100644 --- a/python/composio/client/enums/action.pyi +++ b/python/composio/client/enums/action.pyi @@ -1,7 +1,7 @@ import typing as t import warnings -from composio.client.enums.base import ActionData, replacement_action_name +from composio.client.enums.base import ActionData, create_action from composio.client.enums.enum import Enum, EnumGenerator from composio.constants import VERSION_LATEST, VERSION_LATEST_BASE from composio.exceptions import EnumMetadataNotFound, InvalidVersionString, VersionError @@ -3125,6 +3125,12 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): FIREFLIES_GET_USERS: "Action" FIREFLIES_GET_USER_BY_ID: "Action" FIREFLIES_UPLOAD_AUDIO: "Action" + FOURSQAURE_V2_PLACE_SEARCH: "Action" + FOURSQUARE_V1_FIND_NEARBY_PLACES: "Action" + FOURSQUARE_V1_GET_PLACE_DETAILS: "Action" + FOURSQUARE_V1_GET_PLACE_PHOTOS: "Action" + FOURSQUARE_V1_GET_PLACE_TIPS: "Action" + FOURSQUARE_V1_PLACE_SEARCH: "Action" FRESHDESK_CREATE_TICKET: "Action" FRESHDESK_DELETE_TICKET: "Action" FRESHDESK_GET_TICKETS: "Action" @@ -4088,6 +4094,21 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): GIT_GITHUB_CLONE_CMD: "Action" GIT_GIT_REPO_TREE: "Action" GMAIL_ADD_LABEL_TO_EMAIL: "Action" + GMAIL_BETA_ADD_LABEL_TO_EMAIL: "Action" + GMAIL_BETA_CREATE_EMAIL_DRAFT: "Action" + GMAIL_BETA_CREATE_LABEL: "Action" + GMAIL_BETA_FETCH_EMAILS: "Action" + GMAIL_BETA_FETCH_MESSAGE_BY_MESSAGE_ID: "Action" + GMAIL_BETA_FETCH_MESSAGE_BY_THREAD_ID: "Action" + GMAIL_BETA_GET_ATTACHMENT: "Action" + GMAIL_BETA_GET_PEOPLE: "Action" + GMAIL_BETA_GET_PROFILE: "Action" + GMAIL_BETA_LIST_LABELS: "Action" + GMAIL_BETA_LIST_THREADS: "Action" + GMAIL_BETA_MODIFY_THREAD_LABELS: "Action" + GMAIL_BETA_REMOVE_LABEL: "Action" + GMAIL_BETA_REPLY_TO_THREAD: "Action" + GMAIL_BETA_SEND_EMAIL: "Action" GMAIL_CREATE_EMAIL_DRAFT: "Action" GMAIL_CREATE_LABEL: "Action" GMAIL_FETCH_EMAILS: "Action" @@ -4168,6 +4189,16 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): GOOGLETASKS_LIST_TASK_LISTS: "Action" GOOGLETASKS_PATCH_TASK: "Action" GOOGLETASKS_PATCH_TASK_LIST: "Action" + GOOGLE_DRIVE_BETA_ADD_FILE_SHARING_PREFERENCE: "Action" + GOOGLE_DRIVE_BETA_COPY_FILE: "Action" + GOOGLE_DRIVE_BETA_CREATE_FILE_FROM_TEXT: "Action" + GOOGLE_DRIVE_BETA_CREATE_FOLDER: "Action" + GOOGLE_DRIVE_BETA_DELETE_FOLDER_OR_FILE: "Action" + GOOGLE_DRIVE_BETA_EDIT_FILE: "Action" + GOOGLE_DRIVE_BETA_FIND_FILE: "Action" + GOOGLE_DRIVE_BETA_FIND_FOLDER: "Action" + GOOGLE_DRIVE_BETA_PARSE_FILE: "Action" + GOOGLE_DRIVE_BETA_UPLOAD_FILE: "Action" GOOGLE_MAPS_GET_DIRECTION: "Action" GOOGLE_MAPS_GET_ROUTE: "Action" GOOGLE_MAPS_NEARBY_SEARCH: "Action" @@ -4444,11 +4475,6 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): HUBSPOT_UPDATE_TOKEN_ON_EVENT_TEMPLATE: "Action" HUBSPOT_UPDATE_VIDEO_CONFERENCE_APP_SETTINGS: "Action" IMAGE_ANALYSER_ANALYSE: "Action" - INDUCED_AI_EXTRACT_DATA: "Action" - INDUCED_AI_GET_AUTONOMOUS_TASK_STATUS: "Action" - INDUCED_AI_GET_DATA_EXTRACTION_STATUS: "Action" - INDUCED_AI_PERFORM_AUTONOMOUS_TASK: "Action" - INDUCED_AI_STOP_AUTONOMOUS_TASK: "Action" INTERCOM_ADD_SUBSCRIPTION_TO_A_CONTACT: "Action" INTERCOM_ADD_TAG_TO_A_CONTACT: "Action" INTERCOM_ATTACH_A_CONTACT_TO_A_COMPANY: "Action" @@ -5267,6 +5293,8 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): LINEAR_LIST_LINEAR_LABELS: "Action" LINEAR_LIST_LINEAR_PROJECTS: "Action" LINEAR_LIST_LINEAR_TEAMS: "Action" + LINKEDIN_BETA_GET_RELATIONS: "Action" + LINKEDIN_BETA_SEARCH: "Action" LINKEDIN_CREATE_LINKED_IN_POST: "Action" LINKEDIN_DELETE_LINKED_IN_POST: "Action" LINKEDIN_GET_COMPANY_INFO: "Action" @@ -5763,6 +5791,7 @@ class Action(Enum[ActionData], metaclass=EnumGenerator): NOTION_UPDATE_SCHEMA_DATABASE: "Action" ONEPAGE_SEARCH_INPUT_POST_REQUEST: "Action" ONEPAGE_TOKEN_DETAILS_REQUEST: "Action" + ONE_DRIVE_DOWNLOAD_FILE: "Action" ONE_DRIVE_ONEDRIVE_CREATE_FOLDER: "Action" ONE_DRIVE_ONEDRIVE_CREATE_TEXT_FILE: "Action" ONE_DRIVE_ONEDRIVE_FIND_FILE: "Action" diff --git a/python/composio/client/enums/app.pyi b/python/composio/client/enums/app.pyi index 1ad8565cf9e..e340e5a2d19 100644 --- a/python/composio/client/enums/app.pyi +++ b/python/composio/client/enums/app.pyi @@ -126,6 +126,8 @@ class App(Enum[AppData], metaclass=EnumGenerator): FOMO: "App" FORMCARRY: "App" FORMSITE: "App" + FOURSQAURE_V2: "App" + FOURSQUARE_V1: "App" FRESHBOOKS: "App" FRESHDESK: "App" FRONT: "App" @@ -159,7 +161,6 @@ class App(Enum[AppData], metaclass=EnumGenerator): ICIMS_TALENT_CLOUD: "App" IDEA_SCALE: "App" IMAGE_ANALYSER: "App" - INDUCED_AI: "App" INTERCOM: "App" INTERZOID: "App" JIRA: "App" diff --git a/python/composio/client/enums/base.py b/python/composio/client/enums/base.py index bd145444fe2..7a35f893b68 100644 --- a/python/composio/client/enums/base.py +++ b/python/composio/client/enums/base.py @@ -3,6 +3,7 @@ """ import typing as t +from pathlib import Path from pydantic import Field @@ -24,6 +25,12 @@ ACTIONS_CACHE = LOCAL_CACHE_DIRECTORY / "actions" TRIGGERS_CACHE = LOCAL_CACHE_DIRECTORY / "triggers" +APPS_ETAG = LOCAL_CACHE_DIRECTORY / "apps.etag" +ACTIONS_ETAG = LOCAL_CACHE_DIRECTORY / "actions.etag" +TRIGGERS_ETAG = LOCAL_CACHE_DIRECTORY / "triggers.etag" + +DEPRECATED_MARKER = "< t.List: return list(_runtime_actions) -DEPRECATED_MARKER = "< t.Optional[str]: """If the action is deprecated, get the replacement action name.""" if description is not None and DEPRECATED_MARKER in description: @@ -121,3 +125,19 @@ def replacement_action_name(description: str, app_name: str) -> t.Optional[str]: return (app_name + "_" + newact.replace(">>", "")).upper() return None + + +def create_action(response: dict[str, t.Any], storage_path: Path) -> ActionData: + return ActionData( + name=response["name"], + app=response["appName"], + tags=response["tags"], + no_auth=response["no_auth"], + is_local=False, + is_runtime=False, + shell=False, + path=storage_path, + replaced_by=replacement_action_name( + response["description"], response["appName"] + ), + ) diff --git a/python/composio/client/enums/enum.py b/python/composio/client/enums/enum.py index c075c2227c4..cc219b7ed96 100644 --- a/python/composio/client/enums/enum.py +++ b/python/composio/client/enums/enum.py @@ -100,24 +100,13 @@ def __eq__(self, other: object) -> bool: return False @classmethod - def iter(cls) -> t.Iterator[str]: + def iter(cls) -> t.Iterable[str]: """Yield the enum names as strings.""" - # TODO: fetch trigger names from dedicated endpoint in the future - path = LOCAL_CACHE_DIRECTORY / cls.cache_folder - # If we try to fetch Actions.iter() with local caching disabled - # for example, we'd get here. - if not path.exists(): - # pylint: disable=import-outside-toplevel - from composio.client import Composio - - # pylint: disable=import-outside-toplevel - from composio.client.utils import check_cache_refresh - - check_cache_refresh(Composio.get_latest()) - if not path.exists(): - return - - yield from os.listdir(path) + # pylint: disable=import-outside-toplevel + from composio.client import Composio + + client = Composio.get_latest() + return client.actions.list_enums() @classmethod def all(cls) -> t.Iterator[te.Self]: diff --git a/python/composio/client/enums/tag.pyi b/python/composio/client/enums/tag.pyi index fc5cf51eb93..67eedecfb12 100644 --- a/python/composio/client/enums/tag.pyi +++ b/python/composio/client/enums/tag.pyi @@ -81,6 +81,12 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): AMPLITUDE_USER_PROPERTY: "Tag" AMPLITUDE_USER_ROUTES: "Tag" AMPLITUDE_USER_SEARCH: "Tag" + ANTHROPIC_BASH: "Tag" + ANTHROPIC_COMPUTER: "Tag" + ANTHROPIC_KEYBOARD: "Tag" + ANTHROPIC_MOUSE: "Tag" + ANTHROPIC_SCREENSHOT: "Tag" + ANTHROPIC_WORKSPACE: "Tag" APALEO_IMPORTANT: "Tag" APALEO_PROPERTY: "Tag" APALEO_PROPERTYACTIONS: "Tag" @@ -414,6 +420,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): CLICKUP_USERS: "Tag" CLICKUP_VIEWS: "Tag" CLICKUP_WEBHOOKS: "Tag" + CODE_ANALYSIS_TOOL_INDEX: "Tag" + CODE_FORMAT_TOOL_FORMATTING: "Tag" D2LBRIGHTSPACE_COURSES: "Tag" D2LBRIGHTSPACE_DEMOGRAPHICS: "Tag" D2LBRIGHTSPACE_GRADES: "Tag" @@ -519,6 +527,10 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): ELEVENLABS_VOICES: "Tag" ELEVENLABS_VOICE_GENERATION: "Tag" ELEVENLABS_WORKSPACE: "Tag" + EMBED_TOOL_IMAGE: "Tag" + EMBED_TOOL_INDEXING: "Tag" + EMBED_TOOL_QUERY_IMAGE_EMBEDDINGS: "Tag" + EMBED_TOOL_VECTORSTORE: "Tag" FIGMA_ACTIVITY_LOGS: "Tag" FIGMA_COMMENTS: "Tag" FIGMA_COMMENT_REACTIONS: "Tag" @@ -533,10 +545,14 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): FIGMA_USERS: "Tag" FIGMA_VARIABLES: "Tag" FIGMA_WEBHOOKS: "Tag" + FIRECRAWL_CRAWLING: "Tag" FIRECRAWL_EXTRACTION: "Tag" FIRECRAWL_IMPORTANT: "Tag" FIRECRAWL_SCRAPING: "Tag" FIRECRAWL_WEB: "Tag" + FOURSQAURE_V2_SEARCH___DATA: "Tag" + FOURSQUARE_V1_GEOTAGGING___CHECK_IN: "Tag" + FOURSQUARE_V1_SEARCH___DATA: "Tag" GITHUB_ACTIONS: "Tag" GITHUB_ACTIVITY: "Tag" GITHUB_APPS: "Tag" @@ -573,7 +589,10 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): GITHUB_SECURITY_ADVISORIES: "Tag" GITHUB_TEAMS: "Tag" GITHUB_USERS: "Tag" + GIT_CLI: "Tag" + GMAIL_BETA_IMPORTANT: "Tag" GMAIL_IMPORTANT: "Tag" + GREPTILE_CODE_QUERY: "Tag" HEYGEN_CREATE_VIDEO_API: "Tag" HEYGEN_DEFAULT: "Tag" HEYGEN_IMPORTANT: "Tag" @@ -585,6 +604,7 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): HEYGEN_USER: "Tag" HEYGEN_VIDEO_TRANSLATE_API: "Tag" HEYGEN_WEBHOOKS: "Tag" + HISTORY_FETCHER_WORKSPACE: "Tag" HUBSPOT_ASSET: "Tag" HUBSPOT_AUTOMATION: "Tag" HUBSPOT_BASIC: "Tag" @@ -613,6 +633,7 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): HUBSPOT_TEMPLATES: "Tag" HUBSPOT_TOKENS: "Tag" HUBSPOT_TYPES: "Tag" + IMAGE_ANALYSER_IMAGE: "Tag" INTERCOM_ADMINS: "Tag" INTERCOM_ARTICLES: "Tag" INTERCOM_COMPANIES: "Tag" @@ -769,6 +790,7 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): MAILCHIMP_TEMPLATEFOLDERS: "Tag" MAILCHIMP_TEMPLATES: "Tag" MAILCHIMP_VERIFIEDDOMAINS: "Tag" + MATHEMATICAL_CALCULATOR: "Tag" MEM0_AGENTS: "Tag" MEM0_APPS: "Tag" MEM0_ENTITIES: "Tag" @@ -967,6 +989,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): POSTHOG_SURVEYS: "Tag" POSTHOG_TREND: "Tag" POSTHOG_USERS: "Tag" + RAGTOOL_KNOWLEDGE_BASE: "Tag" + RAGTOOL_RAG: "Tag" SALESFORCE_ACCOUNT: "Tag" SALESFORCE_CAMPAIGN: "Tag" SALESFORCE_CONTACT: "Tag" @@ -1056,6 +1080,9 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): SENTRY_SCIM: "Tag" SENTRY_TEAMS: "Tag" SENTRY_USERS: "Tag" + SHELLTOOL_CREATE: "Tag" + SHELLTOOL_SHELL: "Tag" + SHELLTOOL_WORKSPACE: "Tag" SHOPIFY_IMPORTANT: "Tag" SHOPIFY_PRODUCT_IMAGE: "Tag" SLACKBOT_ADMIN: "Tag" @@ -1172,6 +1199,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): SLACK_USERS_PROFILE: "Tag" SLACK_VIEWS: "Tag" SLACK_WORKFLOWS: "Tag" + SPIDERTOOL_SCRAPE: "Tag" + SPIDERTOOL_WEB: "Tag" SPOTIFY_ALBUMS: "Tag" SPOTIFY_ARTISTS: "Tag" SPOTIFY_AUDIOBOOKS: "Tag" @@ -1188,6 +1217,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): SPOTIFY_SHOWS: "Tag" SPOTIFY_TRACKS: "Tag" SPOTIFY_USERS: "Tag" + SQLTOOL_SQL: "Tag" + SQLTOOL_SQL_QUERY: "Tag" STRIPE_CUSTOMER: "Tag" STRIPE_CUSTOMERS: "Tag" STRIPE_IMPORTANT: "Tag" @@ -1237,6 +1268,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): TWITTER_USERS: "Tag" TYPEFULLY_DRAFTS: "Tag" TYPEFULLY_NOTIFICATIONS: "Tag" + WEBTOOL_WEBBROWSER: "Tag" + WEBTOOL_WEB_BROWSER: "Tag" WRIKE_ACCOUNTS: "Tag" WRIKE_CONTACTS: "Tag" WRIKE_CUSTOM_FIELDS: "Tag" @@ -1247,6 +1280,8 @@ class Tag(Enum[TagData], metaclass=EnumGenerator): WRIKE_TASKS: "Tag" WRIKE_USERS: "Tag" WRIKE_WORKFLOWS: "Tag" + ZEPTOOL_HISTORY: "Tag" + ZEPTOOL_MEMORY: "Tag" ZOHO_CONVERT: "Tag" ZOHO_CREATE: "Tag" ZOHO_FETCH: "Tag" diff --git a/python/composio/client/enums/trigger.pyi b/python/composio/client/enums/trigger.pyi index a5f8a48a459..f6888949723 100644 --- a/python/composio/client/enums/trigger.pyi +++ b/python/composio/client/enums/trigger.pyi @@ -18,14 +18,18 @@ class Trigger(Enum[TriggerData], metaclass=EnumGenerator): @property def app(self) -> str: ... ASANA_TASK_TRIGGER: "Trigger" + DISCORDBOT_DISCORD_NEW_MESSAGE_TRIGGER: "Trigger" + DISCORD_DISCORD_NEW_MESSAGE_TRIGGER: "Trigger" GITHUB_COMMIT_EVENT: "Trigger" GITHUB_FOLLOWER_EVENT: "Trigger" GITHUB_ISSUE_ADDED_EVENT: "Trigger" GITHUB_LABEL_ADDED_EVENT: "Trigger" GITHUB_PULL_REQUEST_EVENT: "Trigger" GITHUB_STAR_ADDED_EVENT: "Trigger" + GMAIL_BETA_NEW_GMAIL_MESSAGE: "Trigger" GMAIL_NEW_GMAIL_MESSAGE: "Trigger" GOOGLEDRIVE_GOOGLE_DRIVE_CHANGES: "Trigger" + GOOGLE_DRIVE_BETA_GOOGLE_DRIVE_CHANGES: "Trigger" HUBSPOT_CONTACT_CREATED_TRIGGER: "Trigger" HUBSPOT_DEAL_STAGE_UPDATED_TRIGGER: "Trigger" HUBSPOT_FORM_SUBMITTED: "Trigger" diff --git a/python/composio/client/http.py b/python/composio/client/http.py index 0435f9993df..1daaa7a084f 100644 --- a/python/composio/client/http.py +++ b/python/composio/client/http.py @@ -77,6 +77,6 @@ def request(url: str, **kwargs: t.Any) -> t.Any: return request def __getattribute__(self, name: str) -> t.Any: - if name in ("get", "post", "put", "delete", "patch"): + if name in ("get", "post", "put", "delete", "patch", "head", "options"): return self._wrap(super().__getattribute__(name)) return super().__getattribute__(name) diff --git a/python/composio/client/utils.py b/python/composio/client/utils.py index b2e811504c8..26ea4c10f19 100644 --- a/python/composio/client/utils.py +++ b/python/composio/client/utils.py @@ -1,10 +1,25 @@ import json import os +import shutil +import threading +import time import typing as t +from pathlib import Path from composio.client import Composio, enums from composio.client.collections import ActionModel, AppModel, TriggerModel -from composio.client.enums.base import replacement_action_name +from composio.client.enums.base import ( + ACTIONS_CACHE, + ACTIONS_ETAG, + APPS_CACHE, + APPS_ETAG, + TAGS_CACHE, + TRIGGERS_CACHE, + TRIGGERS_ETAG, + AppData, + create_action, + replacement_action_name, +) from composio.tools.local import load_local_tools from composio.utils import get_enum_key from composio.utils.logging import get_logger @@ -12,12 +27,26 @@ EnumModels = t.Union[AppModel, ActionModel, TriggerModel] - logger = get_logger(__name__) +_cache_checked = False + NO_CACHE_REFRESH = os.getenv("COMPOSIO_NO_CACHE_REFRESH", "false") == "true" +def _is_update_is_required(file: Path, etag: t.Callable[[], str]) -> bool: + _etag = etag() + if not file.exists(): + file.write_text(_etag) + return True + + if file.read_text() == _etag: + return False + + file.write_text(_etag) + return True + + def filter_non_beta_items(items: t.Sequence[EnumModels]) -> t.List: filtered_items: t.List[EnumModels] = [] for item in items: @@ -34,23 +63,41 @@ def filter_non_beta_items(items: t.Sequence[EnumModels]) -> t.List: return unique_items -def update_apps(client: Composio, beta: bool = False) -> t.List[AppModel]: +def update_apps( + client: Composio, + beta: bool = False, + force: bool = False, +) -> None: """Update apps.""" - apps = sorted( - client.apps.get(), - key=lambda x: x.key, - ) + if not force and not _is_update_is_required( + file=APPS_ETAG, + etag=client.apps.get_etag, + ): + logger.info("Apps cache does not require update!") + return + + logger.info("Updating apps cache...") + shutil.rmtree(APPS_CACHE, ignore_errors=True) + apps = sorted(client.apps.get(), key=lambda x: x.key) if not beta: apps = filter_non_beta_items(apps) - - _update_apps(apps=apps) - return apps + _update_apps_cache(apps=apps) def update_actions( - client: Composio, apps: t.List[AppModel], beta: bool = False + client: Composio, + beta: bool = False, + force: bool = False, ) -> None: """Update actions and tags.""" + if not force and not _is_update_is_required( + file=ACTIONS_ETAG, + etag=client.actions.get_etag, + ): + logger.info("Actions cache does not require update!") + return + + logger.info("Updating actions cache...") actions = sorted( client.actions.get(allow_all=True), key=lambda x: f"{x.appName}_{x.name}", @@ -58,25 +105,36 @@ def update_actions( if not beta: actions = filter_non_beta_items(actions) - _update_tags(apps=apps, actions=actions) - _update_actions(apps=apps, actions=actions) + shutil.rmtree(TAGS_CACHE, ignore_errors=True) + _update_tags_cache(actions=actions) + + shutil.rmtree(ACTIONS_CACHE, ignore_errors=True) + _update_actions_cache(actions=actions) def update_triggers( - client: Composio, apps: t.List[AppModel], beta: bool = False + client: Composio, + beta: bool = False, + force: bool = False, ) -> None: """Update triggers.""" - triggers = sorted( - client.triggers.get(), - key=lambda x: f"{x.appKey}_{x.name}", - ) + if not force and not _is_update_is_required( + file=TRIGGERS_ETAG, + etag=client.triggers.get_etag, + ): + logger.info("Triggers cache does not require update!") + return + + logger.info("Updating triggers cache...") + triggers = sorted(client.triggers.get(), key=lambda x: f"{x.appKey}_{x.name}") if not beta: triggers = filter_non_beta_items(triggers) - _update_triggers(apps=apps, triggers=triggers) + shutil.rmtree(TRIGGERS_CACHE, ignore_errors=True) + _update_triggers_cache(triggers=triggers) -def _update_apps(apps: t.List[AppModel]) -> None: +def _update_apps_cache(apps: t.List[AppModel]) -> None: """Create App enum class.""" app_names = [] enums.base.APPS_CACHE.mkdir( @@ -107,38 +165,34 @@ def _update_apps(apps: t.List[AppModel]) -> None: ).store() -def _update_actions(apps: t.List[AppModel], actions: t.List[ActionModel]) -> None: +def _update_actions_cache(actions: t.List[ActionModel]) -> None: """Get Action enum.""" enums.base.ACTIONS_CACHE.mkdir(parents=True, exist_ok=True) deprecated = {} action_names = [] - for app in sorted(apps, key=lambda x: x.key): - for action in actions: - if action.appName != app.key: - continue - new_action_name = replacement_action_name( - action.description or "", action.appName - ) - if new_action_name is not None: - replaced_by = deprecated[get_enum_key(name=action.name)] = ( - new_action_name - ) - else: - action_names.append(get_enum_key(name=action.name)) - replaced_by = None - - # TODO: there is duplicate ActionData creation code in - # `load_from_runtime` and `fetch_and_cache` in client/enums/action.py - enums.base.ActionData( - name=action.name, - app=app.key, - tags=action.tags, - no_auth=app.no_auth, - is_local=False, - path=enums.base.ACTIONS_CACHE / get_enum_key(name=action.name), - replaced_by=replaced_by, - ).store() + for action in actions: + new_action_name = replacement_action_name( + action.description or "", + action.appName, + ) + if new_action_name is not None: + replaced_by = deprecated[get_enum_key(name=action.name)] = new_action_name + else: + action_names.append(get_enum_key(name=action.name)) + replaced_by = None + + # TODO: there is duplicate ActionData creation code in + # `load_from_runtime` and `fetch_and_cache` in client/enums/action.py + enums.base.ActionData( + name=action.name, + app=action.appName, + tags=action.tags, + no_auth=action.no_auth, + is_local=False, + path=enums.base.ACTIONS_CACHE / get_enum_key(name=action.name), + replaced_by=replaced_by, + ).store() processed = [] for tool in load_local_tools()["local"].values(): @@ -159,16 +213,14 @@ def _update_actions(apps: t.List[AppModel], actions: t.List[ActionModel]) -> Non ).store() -def _update_tags(apps: t.List[AppModel], actions: t.List[ActionModel]) -> None: +def _update_tags_cache(actions: t.List[ActionModel]) -> None: """Create Tag enum class.""" enums.base.TAGS_CACHE.mkdir(parents=True, exist_ok=True) tag_map: t.Dict[str, t.Set[str]] = {} - for app in apps: - app_name = app.key - for action in [action for action in actions if action.appName == app_name]: - if app_name not in tag_map: - tag_map[app_name] = set() - tag_map[app_name].update(action.tags or []) + for action in actions: + if action.appName not in tag_map: + tag_map[action.appName] = set() + tag_map[action.appName].update(action.tags or []) tag_names = ["DEFAULT"] for app_name in sorted(tag_map): @@ -188,24 +240,82 @@ def _update_tags(apps: t.List[AppModel], actions: t.List[ActionModel]) -> None: ) -def _update_triggers( - apps: t.List[AppModel], - triggers: t.List[TriggerModel], -) -> None: +def _update_triggers_cache(triggers: t.List[TriggerModel]) -> None: """Get Trigger enum.""" - trigger_names = [] enums.base.TRIGGERS_CACHE.mkdir(exist_ok=True) - for app in apps: - for trigger in triggers: - if trigger.appKey != app.key: + for trigger in triggers: + enums.base.TriggerData( + name=trigger.name, + app=trigger.appKey, + path=enums.base.TRIGGERS_CACHE / get_enum_key(name=trigger.name), + ).store() + + +def _check_and_refresh_actions(client: Composio): + local_actions = set() + if enums.base.ACTIONS_CACHE.exists(): + for action in enums.base.ACTIONS_CACHE.iterdir(): + action_data = json.loads(action.read_text()) + # The action file could be old. If it doesn't have a + # replaced_by field, we want to overwrite it. + if "replaced_by" not in action_data: + action.unlink() continue + local_actions.add(action.stem.upper()) - trigger_names.append(get_enum_key(name=trigger.name).upper()) - enums.base.TriggerData( - name=trigger.name, - app=app.key, - path=enums.base.TRIGGERS_CACHE / trigger_names[-1], - ).store() + api_actions = client.actions.list_enums() + actions_to_update = set(api_actions) - set(local_actions) + actions_to_delete = set(local_actions) - set(api_actions) + if actions_to_delete: + logger.debug("Stale actions: %s", actions_to_delete) + + for action_name in actions_to_delete: + (enums.base.ACTIONS_CACHE / action_name).unlink() + + if not actions_to_update: + return + + logger.debug( + "Actions to fetch: %s %s...", + str(len(actions_to_update)), + str(actions_to_update)[:64], + ) + queries = {"actions": ",".join(actions_to_update)} + actions_request = client.http.get(url=str(client.actions.endpoint(queries))) + if actions_request.status_code == 414: + actions_request = client.http.get(url=str(client.actions.endpoint({}))) + + actions_data = actions_request.json() + for action_data in actions_data["items"]: + create_action( + response=action_data, + storage_path=enums.base.ACTIONS_CACHE / action_data["name"], + ).store() + + +def _check_and_refresh_apps(client: Composio): + local_apps = set() + if enums.base.APPS_CACHE.exists(): + local_apps = set(map(lambda x: x.stem.upper(), enums.base.APPS_CACHE.iterdir())) + + api_apps = set(client.apps.list_enums()) + apps_to_update = api_apps - local_apps + apps_to_delete = local_apps - api_apps + if apps_to_delete: + logger.debug("Stale apps: %s", apps_to_delete) + + for app_name in apps_to_delete: + (enums.base.APPS_CACHE / app_name).unlink() + + if not apps_to_update: + return + + logger.debug("Apps to fetch: %s", apps_to_update) + queries = {"apps": ",".join(apps_to_update)} + apps_data = client.http.get(str(client.apps.endpoint(queries))).json() + for app_data in apps_data["items"]: + storage_path = enums.base.APPS_CACHE / app_data["name"] + AppData(name=app_data["name"], path=storage_path, is_local=False).store() def check_cache_refresh(client: Composio) -> None: @@ -221,15 +331,19 @@ def check_cache_refresh(client: Composio) -> None: if NO_CACHE_REFRESH: return - if enums.base.ACTIONS_CACHE.exists(): - first_file = next(enums.base.ACTIONS_CACHE.iterdir(), None) - if first_file is not None: - first_action = json.loads(first_file.read_text()) - if "replaced_by" in first_action: - logger.debug("Actions cache is up-to-date") - return - - logger.info("Actions cache is outdated, refreshing cache...") - apps = update_apps(client) - update_actions(client, apps) - update_triggers(client, apps) + global _cache_checked + if _cache_checked: + return + _cache_checked = True + + logger.debug("Checking cache...") + start = time.monotonic() + ap_thread = threading.Thread(target=_check_and_refresh_apps, args=(client,)) + ac_thread = threading.Thread(target=_check_and_refresh_actions, args=(client,)) + + ac_thread.start() + ap_thread.start() + + ap_thread.join() + ac_thread.join() + logger.debug("Time taken to update cache: %.2f seconds", time.monotonic() - start) diff --git a/python/composio/server/api.py b/python/composio/server/api.py index 744db91c98d..38b24d62260 100644 --- a/python/composio/server/api.py +++ b/python/composio/server/api.py @@ -173,9 +173,9 @@ def _update_apps() -> bool: update_triggers, ) - apps = update_apps(client=get_context().client) - update_actions(client=get_context().client, apps=apps) - update_triggers(client=get_context().client, apps=apps) + update_apps(client=get_context().client) + update_actions(client=get_context().client) + update_triggers(client=get_context().client) return True @app.get("/api/apps/{name}", response_model=APIResponse[AppModel]) diff --git a/python/composio/tools/local/codeanalysis/tool.py b/python/composio/tools/local/codeanalysis/tool.py index 732ae18a302..37a4b4ecd36 100644 --- a/python/composio/tools/local/codeanalysis/tool.py +++ b/python/composio/tools/local/codeanalysis/tool.py @@ -21,6 +21,7 @@ class CodeAnalysisTool(LocalTool, autoload=True): "tree_sitter_python>=0.22.0", "git+https://github.com/DataDog/jedi.git@92d0c807b0dcd115b1ffd0a4ed21e44db127c2fb#egg=jedi", "PyJWT", # deeplake/client/client.py:41 + "azure<5.0.0", ] logo = "https://raw.githubusercontent.com/ComposioHQ/composio/master/python/docs/imgs/logos/codemap.png" diff --git a/python/composio/tools/toolset.py b/python/composio/tools/toolset.py index c5b61bbcf05..b7c187db204 100644 --- a/python/composio/tools/toolset.py +++ b/python/composio/tools/toolset.py @@ -48,7 +48,6 @@ from composio.client.enums import TriggerType from composio.client.exceptions import ComposioClientError, HTTPError, NoItemsFound from composio.client.files import FileDownloadable, FileUploadable -from composio.client.utils import check_cache_refresh from composio.constants import ( DEFAULT_ENTITY_ID, ENV_COMPOSIO_API_KEY, @@ -481,6 +480,7 @@ def get_runtime_action_schemas( items.append( ActionModel( **schema, + no_auth=True, version=VERSION_LATEST, available_versions=[VERSION_LATEST], ).model_copy(deep=True) @@ -504,6 +504,7 @@ def get_local_action_schemas( items = [ ActionModel( **item, + no_auth=True, version=VERSION_LATEST, available_versions=[VERSION_LATEST], ) @@ -1594,7 +1595,6 @@ def _init_client(self) -> Composio: base_url=self._base_url, runtime=self._runtime, ) - check_cache_refresh(self._remote_client) self._remote_client.local = self._local_client return self._remote_client diff --git a/python/tests/test_tools/test_toolset.py b/python/tests/test_tools/test_toolset.py index 69e95c56925..38a3ccfdd22 100644 --- a/python/tests/test_tools/test_toolset.py +++ b/python/tests/test_tools/test_toolset.py @@ -3,7 +3,6 @@ """ import logging -import os import re import typing as t from unittest import mock @@ -49,9 +48,7 @@ def test_get_trigger_config_scheme() -> None: def test_delete_trigger() -> None: """Test `ComposioToolSet.delete_trigger` method.""" - api_key = os.getenv("COMPOSIO_API_KEY") - toolset = ComposioToolSet(api_key=api_key) - + toolset = ComposioToolSet() connected_account_id: str for account in toolset.get_connected_accounts(): if account.appName == "gmail": @@ -61,9 +58,12 @@ def test_delete_trigger() -> None: enabled_trigger = toolset.client.triggers.enable( name="GMAIL_NEW_GMAIL_MESSAGE", connected_account_id=connected_account_id, - config={"interval": 1, "userId": "me", "labelIds": "INBOX"}, + config={ + "interval": 1, + "userId": "me", + "labelIds": "INBOX", + }, ) - assert enabled_trigger["triggerId"] is not None assert toolset.delete_trigger(id=enabled_trigger["triggerId"]) is True @@ -75,7 +75,9 @@ def test_find_actions_by_tags() -> None: assert "important" in action.tags for action in toolset.find_actions_by_tags( - App.SLACK, App.GITHUB, tags=["important"] + App.SLACK, + App.GITHUB, + tags=["important"], ): assert "important" in action.tags assert action.app in ("GITHUB", "SLACK", "SLACKBOT") diff --git a/python/tox.ini b/python/tox.ini index 576da38d263..8a47d4f1760 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -101,6 +101,7 @@ deps = libcst # codeanalysis sentence_transformers # codeanalysis tree_sitter_python>=0.22.0 # codeanalysis + azure<5.0.0 # codeanalysis PyJWT # deeplake/client/client.py:41 e2b>=0.17.2a37 # E2B Workspace e2b-code-interpreter # E2B workspace