diff --git a/.gitignore b/.gitignore index 84e9db6f..4bb0d929 100644 --- a/.gitignore +++ b/.gitignore @@ -168,4 +168,7 @@ cython_debug/ *.agent-catalog/ *.agent-activity/ *.model-cache/ -*.data/ \ No newline at end of file +*.data/ + +# Security certificate files +*.pem \ No newline at end of file diff --git a/libs/agentc/agentc/auditor.py b/libs/agentc/agentc/auditor.py index ac72e16a..7f820935 100644 --- a/libs/agentc/agentc/auditor.py +++ b/libs/agentc/agentc/auditor.py @@ -71,10 +71,12 @@ class Auditor(pydantic_settings.BaseSettings): This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`. """ - certificate: typing.Optional[str] = None + conn_root_certificate: typing.Optional[str | pathlib.Path] = None """ Path to the root certificate for TLS associated with the Couchbase cluster. - This field **can** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password` for secure connection verification with Couchbase cluster. + This field is optional and only required if the Couchbase cluster is using a self-signed certificate. + If specified, this field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, + and :py:attr:`password`. """ catalog: typing.Optional[pathlib.Path] = None @@ -169,11 +171,13 @@ def _initialize_auditor(self) -> typing.Self: if self.conn_string is not None: try: + if self.conn_root_certificate is not None and isinstance(self.conn_root_certificate, pathlib.Path): + self.conn_root_certificate = self.conn_root_certificate.absolute() self._db_auditor = DBAuditor( conn_string=self.conn_string, username=self.username.get_secret_value(), password=self.password.get_secret_value(), - certificate=self.certificate, + certificate=self.conn_root_certificate, model_name=self.llm_model_name, agent_name=self.agent_name, bucket=self.bucket, diff --git a/libs/agentc/agentc/provider.py b/libs/agentc/agentc/provider.py index c06df097..d1354be8 100644 --- a/libs/agentc/agentc/provider.py +++ b/libs/agentc/agentc/provider.py @@ -55,25 +55,25 @@ class Provider(pydantic_settings.BaseSettings): If this field AND ``$AGENT_CATALOG_CATALOG`` are specified, we will issue :command:`find` on both the remote and local catalog (with local catalog entries taking precedence). - This field must be specified with :py:attr:`username`, :py:attr:`password`, and :py:attr:`bucket`. + This field **must** be specified with :py:attr:`username`, :py:attr:`password`, and :py:attr:`bucket`. """ username: typing.Optional[pydantic.SecretStr] = None """ Username associated with the Couchbase instance possessing the catalog. - This field must be specified with :py:attr:`conn_string`, :py:attr:`password`, and :py:attr:`bucket`. + This field **must** be specified with :py:attr:`conn_string`, :py:attr:`password`, and :py:attr:`bucket`. """ password: typing.Optional[pydantic.SecretStr] = None """ Password associated with the Couchbase instance possessing the catalog. - This field must be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`bucket`. + This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`bucket`. """ bucket: typing.Optional[str] = None """ The name of the Couchbase bucket possessing the catalog. - This field must be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`. + This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`. """ catalog: typing.Optional[pathlib.Path] = None @@ -83,6 +83,14 @@ class Provider(pydantic_settings.BaseSettings): from the current working directory until we find the :py:data:`agentc_core.defaults.DEFAULT_ACTIVITY_FOLDER` folder. """ + conn_root_certificate: typing.Optional[str | pathlib.Path] = None + """ Path to the root certificate file for the Couchbase cluster. + + This field is optional and only required if the Couchbase cluster is using a self-signed certificate. + If specified, this field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, + and :py:attr:`password`. + """ + snapshot: typing.Optional[str] = LATEST_SNAPSHOT_VERSION """ The snapshot version to find the tools and prompts for. @@ -218,12 +226,15 @@ def _find_remote_catalog(self) -> typing.Self: # Try to connect to our cluster. try: + if self.conn_root_certificate is not None and isinstance(self.conn_root_certificate, pathlib.Path): + self.conn_root_certificate = self.conn_root_certificate.absolute() cluster = couchbase.cluster.Cluster.connect( self.conn_string, couchbase.options.ClusterOptions( couchbase.auth.PasswordAuthenticator( username=self.username.get_secret_value(), password=self.password.get_secret_value(), + cert_path=self.conn_root_certificate, ) ), ) diff --git a/libs/agentc_cli/tests/test_click.py b/libs/agentc_cli/tests/test_click.py index 6d59f24c..3e548185 100644 --- a/libs/agentc_cli/tests/test_click.py +++ b/libs/agentc_cli/tests/test_click.py @@ -51,14 +51,14 @@ def test_index(tmp_path): shutil.copy(resources_folder / "_good_spec.json", tool_folder / "_good_spec.json") invocation = runner.invoke(click_main, ["index", str(tool_folder.absolute()), "--no-prompts"]) - # We should see 10 files scanned and 11 tools indexed. + # We should see 11 files scanned and 12 tools indexed. output = invocation.output print(output) assert "Crawling" in output assert "Generating embeddings" in output assert "Catalog successfully indexed" in output - assert "0/10" in output assert "0/11" in output + assert "0/12" in output # Small helper function to publish to a Couchbase catalog. diff --git a/libs/agentc_core/agentc_core/analytics/ddls/tool_calls.sqlpp b/libs/agentc_core/agentc_core/analytics/ddls/tool_calls.sqlpp index 95315640..8f97465c 100644 --- a/libs/agentc_core/agentc_core/analytics/ddls/tool_calls.sqlpp +++ b/libs/agentc_core/agentc_core/analytics/ddls/tool_calls.sqlpp @@ -2,12 +2,12 @@ CREATE OR REPLACE ANALYTICS VIEW `[BUCKET_NAME]`.`[SCOPE_NAME]`.ToolCalls AS FROM -- The tool calls our LLM has authored. - Sessions AS s1, + `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s1, s1.msgs AS m1, -- Note: this is LangChain specific! We need to factor this out later. m1.content.dump.kwargs.tool_calls AS tc1, -- The results of the tools. - Sessions AS s2, + `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s2, s2.msgs AS m2 WHERE m1.kind = "llm" AND diff --git a/libs/agentc_core/agentc_core/analytics/ddls/trajectories.sqlpp b/libs/agentc_core/agentc_core/analytics/ddls/trajectories.sqlpp index 27b9a08d..d60b2ec3 100644 --- a/libs/agentc_core/agentc_core/analytics/ddls/trajectories.sqlpp +++ b/libs/agentc_core/agentc_core/analytics/ddls/trajectories.sqlpp @@ -1,7 +1,7 @@ -- Note: all_sessions.sqlpp should be run before this script. CREATE OR REPLACE ANALYTICS VIEW `[BUCKET_NAME]`.`[SCOPE_NAME]`.Walks AS FROM - Sessions AS s, + `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s, s.msgs AS msg WHERE msg.kind = "transition" diff --git a/libs/agentc_core/agentc_core/tool/descriptor/secrets.py b/libs/agentc_core/agentc_core/tool/descriptor/secrets.py index 5c0a697a..b6838e83 100644 --- a/libs/agentc_core/agentc_core/tool/descriptor/secrets.py +++ b/libs/agentc_core/agentc_core/tool/descriptor/secrets.py @@ -1,4 +1,5 @@ import pydantic +import typing # TODO (GLENN): Add more types of secrets below. @@ -7,5 +8,6 @@ class Couchbase(pydantic.BaseModel): conn_string: str username: str password: str + certificate: typing.Optional[str] = None couchbase: Couchbase diff --git a/libs/agentc_core/agentc_core/tool/generate/templates/httpreq_q.jinja b/libs/agentc_core/agentc_core/tool/generate/templates/httpreq_q.jinja index 2ee38a91..810e9ac2 100644 --- a/libs/agentc_core/agentc_core/tool/generate/templates/httpreq_q.jinja +++ b/libs/agentc_core/agentc_core/tool/generate/templates/httpreq_q.jinja @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) {% if input is not none %} {{ input.model.generated_code | safe }} -parameter_locations = {{ input.locations }} +parameter_locations = {{ input.locations | safe }} {% endif %} @tool @@ -39,7 +39,7 @@ def {{ tool.name }}({% if input is not none %}argument_input: {{ input.model.typ {% endif %} # TODO (GLENN): We need to formalize how to handle multiple URLs. - for server_url in {{ urls }}: + for server_url in {{ urls | safe }}: url = server_url + '{{ path }}' request_body = dict() parameters = dict() diff --git a/libs/agentc_core/agentc_core/tool/generate/templates/semantic_q.jinja b/libs/agentc_core/agentc_core/tool/generate/templates/semantic_q.jinja index edad90be..cdb84343 100644 --- a/libs/agentc_core/agentc_core/tool/generate/templates/semantic_q.jinja +++ b/libs/agentc_core/agentc_core/tool/generate/templates/semantic_q.jinja @@ -22,7 +22,8 @@ logger = logging.getLogger(__name__) def _get_couchbase_cluster() -> couchbase.cluster.Cluster: authenticator = couchbase.auth.PasswordAuthenticator( username=get_secret("{{ secrets.username }}").get_secret_value(), - password=get_secret("{{ secrets.password }}").get_secret_value() + password=get_secret("{{ secrets.password }}").get_secret_value(), + {% if secrets.certificate is none %}cert_path=get_secret("{{ secrets.certificate | safe }}").get_secret_value(){% endif %} ) conn_string = get_secret("{{ secrets.conn_string }}").get_secret_value() return couchbase.cluster.Cluster(conn_string, couchbase.options.ClusterOptions(authenticator)) diff --git a/libs/agentc_core/agentc_core/tool/generate/templates/sqlpp_q.jinja b/libs/agentc_core/agentc_core/tool/generate/templates/sqlpp_q.jinja index 0652fd80..9fd58b2f 100644 --- a/libs/agentc_core/agentc_core/tool/generate/templates/sqlpp_q.jinja +++ b/libs/agentc_core/agentc_core/tool/generate/templates/sqlpp_q.jinja @@ -18,7 +18,8 @@ logger = logging.getLogger(__name__) def _get_couchbase_cluster() -> couchbase.cluster.Cluster: authenticator = couchbase.auth.PasswordAuthenticator( username=get_secret("{{ secrets.username }}").get_secret_value(), - password=get_secret("{{ secrets.password }}").get_secret_value() + password=get_secret("{{ secrets.password }}").get_secret_value(), + {% if secrets.certificate is not none %}cert_path=get_secret("{{ secrets.certificate | safe }}").get_secret_value(){% endif %} ) conn_string = get_secret("{{ secrets.conn_string }}").get_secret_value() return couchbase.cluster.Cluster(conn_string, couchbase.options.ClusterOptions(authenticator)) diff --git a/libs/agentc_core/agentc_core/tool/templates/semantic_search.jinja b/libs/agentc_core/agentc_core/tool/templates/semantic_search.jinja index d9a28e5a..bf46c323 100644 --- a/libs/agentc_core/agentc_core/tool/templates/semantic_search.jinja +++ b/libs/agentc_core/agentc_core/tool/templates/semantic_search.jinja @@ -28,6 +28,7 @@ secrets: conn_string: CB_CONN_STRING username: CB_USERNAME password: CB_PASSWORD + certificate: CB_CERTIFICATE # Couchbase semantic search tools always involve a vector search. vector_search: diff --git a/libs/agentc_core/agentc_core/tool/templates/sqlpp_query.jinja b/libs/agentc_core/agentc_core/tool/templates/sqlpp_query.jinja index 90a2c659..30d3037b 100644 --- a/libs/agentc_core/agentc_core/tool/templates/sqlpp_query.jinja +++ b/libs/agentc_core/agentc_core/tool/templates/sqlpp_query.jinja @@ -36,5 +36,6 @@ secrets: conn_string: CB_CONN_STRING username: CB_USERNAME password: CB_PASSWORD + certificate: CB_CERTIFICATE */ <<< Replace me with your SQL++ query! >>> \ No newline at end of file diff --git a/libs/agentc_core/agentc_core/util/ddl.py b/libs/agentc_core/agentc_core/util/ddl.py index 74689f04..0094c394 100644 --- a/libs/agentc_core/agentc_core/util/ddl.py +++ b/libs/agentc_core/agentc_core/util/ddl.py @@ -310,7 +310,7 @@ def create_vector_index( def create_gsi_indexes(bucket, cluster, kind, print_progress): """Creates required indexes at publish""" - progress_bar = tqdm.tqdm(range(4)) + progress_bar = tqdm.tqdm(range(5)) progress_bar_it = iter(progress_bar) completion_status = True @@ -332,6 +332,22 @@ def create_gsi_indexes(bucket, cluster, kind, print_progress): all_errs += err completion_status = False + # Primary index on kind_metadata + primary_idx_metadata_name = f"v1_agent_catalog_primary_{kind}_metadata" + primary_idx = f""" + CREATE PRIMARY INDEX IF NOT EXISTS `{primary_idx_metadata_name}` + ON `{bucket}`.`{DEFAULT_CATALOG_SCOPE}`.`{kind}_metadata` USING GSI; + """ + if print_progress: + next(progress_bar_it) + progress_bar.set_description(primary_idx_name) + res, err = execute_query(cluster, primary_idx) + for r in res.rows(): + logger.debug(r) + if err is not None: + all_errs += err + completion_status = False + # Secondary index on catalog_identifier cat_idx_name = f"v1_agent_catalog_{kind}cat_version_identifier" cat_idx = f""" diff --git a/libs/agentc_core/tests/tool/resources/sqlpp_query/positive_4.sqlpp b/libs/agentc_core/tests/tool/resources/sqlpp_query/positive_4.sqlpp new file mode 100644 index 00000000..0cf87283 --- /dev/null +++ b/libs/agentc_core/tests/tool/resources/sqlpp_query/positive_4.sqlpp @@ -0,0 +1,26 @@ +/* +name: tool_4 + +record_kind: sqlpp_query + +description: > + i am a dummy tool + hello i am a dummy tool + +input: > + { + "type": "object", + "properties": { + "source_airport": { "type": "string" }, + "destination_airport": { "type": "string" } + } + } + +secrets: + - couchbase: + conn_string: CB_CONN_STRING + username: CB_USERNAME + password: CB_PASSWORD + certificate: CB_CERTIFICATE +*/ +SELECT 1; diff --git a/templates/agents/with_controlflow/.env.example b/templates/agents/with_controlflow/.env.example index e4704b13..3a80bb36 100644 --- a/templates/agents/with_controlflow/.env.example +++ b/templates/agents/with_controlflow/.env.example @@ -4,8 +4,9 @@ AGENT_CATALOG_USERNAME=Administrator AGENT_CATALOG_PASSWORD=password AGENT_CATALOG_BUCKET=travel-sample -# In case of capella instance or if secure connection is required -# replace couchbase with couchbases in AGENT_CATALOG_CONN_STRING and add the following +# For Capella instances or if a security certificate is required... +# ...replace 'couchbase' with 'couchbases' in AGENT_CATALOG_CONN_STRING above... +# ...and modify the line below to point to the root certificate on your local system. # AGENT_CATALOG_CONN_ROOT_CERTIFICATE=/path/to/cluster/root/certificate/on/local/system # Couchbase-specific environment variables (for the travel-agent example tools). diff --git a/templates/agents/with_controlflow/.gitignore b/templates/agents/with_controlflow/.gitignore index 3dc84fe4..ed6ff352 100644 --- a/templates/agents/with_controlflow/.gitignore +++ b/templates/agents/with_controlflow/.gitignore @@ -5,3 +5,4 @@ __pycache__ .env poetry.lock +*.pem \ No newline at end of file diff --git a/templates/agents/with_controlflow/agent.py b/templates/agents/with_controlflow/agent.py index b24a9b1a..f4eaa2f6 100644 --- a/templates/agents/with_controlflow/agent.py +++ b/templates/agents/with_controlflow/agent.py @@ -12,7 +12,6 @@ import pydantic import uuid -from pydantic import SecretStr from utils import TaskFactory # Make sure you populate your .env file with the correct credentials! @@ -29,9 +28,10 @@ # The 'values' of this dictionary map to actual values required by the tool. # In this case, we get the Couchbase connection string, username, and password from environment variables. secrets={ - "CB_CONN_STRING": SecretStr(os.getenv("CB_CONN_STRING")), - "CB_USERNAME": SecretStr(os.getenv("CB_USERNAME")), - "CB_PASSWORD": SecretStr(os.getenv("CB_PASSWORD")), + "CB_CONN_STRING": os.getenv("CB_CONN_STRING"), + "CB_USERNAME": os.getenv("CB_USERNAME"), + "CB_PASSWORD": os.getenv("CB_PASSWORD"), + "CB_CERTIFICATE": os.getenv("CB_CERTIFICATE"), }, ) diff --git a/templates/agents/with_controlflow/tools/find_direct_flights.sqlpp b/templates/agents/with_controlflow/tools/find_direct_flights.sqlpp index 76c544c3..47c4bd16 100644 --- a/templates/agents/with_controlflow/tools/find_direct_flights.sqlpp +++ b/templates/agents/with_controlflow/tools/find_direct_flights.sqlpp @@ -57,6 +57,7 @@ secrets: conn_string: CB_CONN_STRING username: CB_USERNAME password: CB_PASSWORD + certificate: CB_CERTIFICATE */ FROM diff --git a/templates/agents/with_controlflow/tools/find_one_layover_flights.sqlpp b/templates/agents/with_controlflow/tools/find_one_layover_flights.sqlpp index cf3fb431..d210ec08 100644 --- a/templates/agents/with_controlflow/tools/find_one_layover_flights.sqlpp +++ b/templates/agents/with_controlflow/tools/find_one_layover_flights.sqlpp @@ -58,6 +58,7 @@ secrets: conn_string: CB_CONN_STRING username: CB_USERNAME password: CB_PASSWORD + certificate: CB_CERTIFICATE */ FROM