better certificate support

.gitignore

@@ -168,4 +168,7 @@ cython_debug/
 *.agent-catalog/
 *.agent-activity/
 *.model-cache/
-*.data/
+*.data/
+
+# Security certificate files
+*.pem

libs/agentc/agentc/auditor.py

@@ -71,10 +71,12 @@ class Auditor(pydantic_settings.BaseSettings):
     This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`.
     """
 
-    certificate: typing.Optional[str] = None
+    conn_root_certificate: typing.Optional[str | pathlib.Path] = None
     """ Path to the root certificate for TLS associated with the Couchbase cluster.
 
-    This field **can** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password` for secure connection verification with Couchbase cluster.
+    This field is optional and only required if the Couchbase cluster is using a self-signed certificate.
+    If specified, this field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`,
+    and :py:attr:`password`.
     """
 
     catalog: typing.Optional[pathlib.Path] = None
@@ -169,11 +171,13 @@ def _initialize_auditor(self) -> typing.Self:
 
         if self.conn_string is not None:
             try:
+                if self.conn_root_certificate is not None and isinstance(self.conn_root_certificate, pathlib.Path):
+                    self.conn_root_certificate = self.conn_root_certificate.absolute()
                 self._db_auditor = DBAuditor(
                     conn_string=self.conn_string,
                     username=self.username.get_secret_value(),
                     password=self.password.get_secret_value(),
-                    certificate=self.certificate,
+                    certificate=self.conn_root_certificate,
                     model_name=self.llm_model_name,
                     agent_name=self.agent_name,
                     bucket=self.bucket,

libs/agentc/agentc/provider.py

@@ -55,25 +55,25 @@ class Provider(pydantic_settings.BaseSettings):
     If this field AND ``$AGENT_CATALOG_CATALOG`` are specified, we will issue :command:`find` on both the remote and
     local catalog (with local catalog entries taking precedence).
 
-    This field must be specified with :py:attr:`username`, :py:attr:`password`, and  :py:attr:`bucket`.
+    This field **must** be specified with :py:attr:`username`, :py:attr:`password`, and  :py:attr:`bucket`.
     """
 
     username: typing.Optional[pydantic.SecretStr] = None
     """ Username associated with the Couchbase instance possessing the catalog.
 
-    This field must be specified with :py:attr:`conn_string`, :py:attr:`password`, and :py:attr:`bucket`.
+    This field **must** be specified with :py:attr:`conn_string`, :py:attr:`password`, and :py:attr:`bucket`.
     """
 
     password: typing.Optional[pydantic.SecretStr] = None
     """ Password associated with the Couchbase instance possessing the catalog.
 
-    This field must be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`bucket`.
+    This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`bucket`.
     """
 
     bucket: typing.Optional[str] = None
     """ The name of the Couchbase bucket possessing the catalog.
 
-    This field must be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`.
+    This field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`, and :py:attr:`password`.
     """
 
     catalog: typing.Optional[pathlib.Path] = None
@@ -83,6 +83,14 @@ class Provider(pydantic_settings.BaseSettings):
     from the current working directory until we find the :py:data:`agentc_core.defaults.DEFAULT_ACTIVITY_FOLDER` folder.
     """
 
+    conn_root_certificate: typing.Optional[str | pathlib.Path] = None
+    """ Path to the root certificate file for the Couchbase cluster.
+
+    This field is optional and only required if the Couchbase cluster is using a self-signed certificate.
+    If specified, this field **must** be specified with :py:attr:`conn_string`, :py:attr:`username`,
+    and :py:attr:`password`.
+    """
+
     snapshot: typing.Optional[str] = LATEST_SNAPSHOT_VERSION
     """ The snapshot version to find the tools and prompts for.
 
@@ -218,12 +226,15 @@ def _find_remote_catalog(self) -> typing.Self:
 
         # Try to connect to our cluster.
         try:
+            if self.conn_root_certificate is not None and isinstance(self.conn_root_certificate, pathlib.Path):
+                self.conn_root_certificate = self.conn_root_certificate.absolute()
             cluster = couchbase.cluster.Cluster.connect(
                 self.conn_string,
                 couchbase.options.ClusterOptions(
                     couchbase.auth.PasswordAuthenticator(
                         username=self.username.get_secret_value(),
                         password=self.password.get_secret_value(),
+                        cert_path=self.conn_root_certificate,
                     )
                 ),
             )

libs/agentc_cli/tests/test_click.py

@@ -51,14 +51,14 @@ def test_index(tmp_path):
         shutil.copy(resources_folder / "_good_spec.json", tool_folder / "_good_spec.json")
         invocation = runner.invoke(click_main, ["index", str(tool_folder.absolute()), "--no-prompts"])
 
-        # We should see 10 files scanned and 11 tools indexed.
+        # We should see 11 files scanned and 12 tools indexed.
         output = invocation.output
         print(output)
         assert "Crawling" in output
         assert "Generating embeddings" in output
         assert "Catalog successfully indexed" in output
-        assert "0/10" in output
         assert "0/11" in output
+        assert "0/12" in output
 
 
 # Small helper function to publish to a Couchbase catalog.

libs/agentc_core/agentc_core/analytics/ddls/tool_calls.sqlpp

@@ -2,12 +2,12 @@
 CREATE OR REPLACE ANALYTICS VIEW `[BUCKET_NAME]`.`[SCOPE_NAME]`.ToolCalls AS
 FROM
     -- The tool calls our LLM has authored.
-    Sessions AS s1,
+    `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s1,
     s1.msgs AS m1,
     -- Note: this is LangChain specific! We need to factor this out later.
     m1.content.dump.kwargs.tool_calls AS tc1,
     -- The results of the tools.
-    Sessions AS s2,
+    `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s2,
     s2.msgs AS m2
 WHERE
     m1.kind = "llm" AND

libs/agentc_core/agentc_core/analytics/ddls/trajectories.sqlpp

@@ -1,7 +1,7 @@
 -- Note: all_sessions.sqlpp should be run before this script.
 CREATE OR REPLACE ANALYTICS VIEW `[BUCKET_NAME]`.`[SCOPE_NAME]`.Walks AS
 FROM
-    Sessions AS s,
+    `[BUCKET_NAME]`.`[SCOPE_NAME]`.Sessions AS s,
     s.msgs AS msg
 WHERE
     msg.kind = "transition"

libs/agentc_core/agentc_core/tool/descriptor/secrets.py

@@ -1,4 +1,5 @@
 import pydantic
+import typing
 
 
 # TODO (GLENN): Add more types of secrets below.
@@ -7,5 +8,6 @@ class Couchbase(pydantic.BaseModel):
         conn_string: str
         username: str
         password: str
+        certificate: typing.Optional[str] = None
 
     couchbase: Couchbase

libs/agentc_core/agentc_core/tool/generate/templates/httpreq_q.jinja

@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
 
 {% if input is not none %}
 {{ input.model.generated_code | safe }}
-parameter_locations = {{ input.locations }}
+parameter_locations = {{ input.locations | safe }}
 {% endif %}
 
 @tool
@@ -39,7 +39,7 @@ def {{ tool.name }}({% if input is not none %}argument_input: {{ input.model.typ
     {% endif %}
 
     # TODO (GLENN): We need to formalize how to handle multiple URLs.
-    for server_url in {{ urls }}:
+    for server_url in {{ urls | safe }}:
         url = server_url + '{{ path }}'
         request_body = dict()
         parameters = dict()

libs/agentc_core/agentc_core/tool/generate/templates/semantic_q.jinja

@@ -22,7 +22,8 @@ logger = logging.getLogger(__name__)
 def _get_couchbase_cluster() -> couchbase.cluster.Cluster:
     authenticator = couchbase.auth.PasswordAuthenticator(
         username=get_secret("{{ secrets.username }}").get_secret_value(),
-        password=get_secret("{{ secrets.password }}").get_secret_value()
+        password=get_secret("{{ secrets.password }}").get_secret_value(),
+        {% if secrets.certificate is none %}cert_path=get_secret("{{ secrets.certificate | safe }}").get_secret_value(){% endif %}
     )
     conn_string = get_secret("{{ secrets.conn_string }}").get_secret_value()
     return couchbase.cluster.Cluster(conn_string, couchbase.options.ClusterOptions(authenticator))

libs/agentc_core/agentc_core/tool/generate/templates/sqlpp_q.jinja

@@ -18,7 +18,8 @@ logger = logging.getLogger(__name__)
 def _get_couchbase_cluster() -> couchbase.cluster.Cluster:
     authenticator = couchbase.auth.PasswordAuthenticator(
         username=get_secret("{{ secrets.username }}").get_secret_value(),
-        password=get_secret("{{ secrets.password }}").get_secret_value()
+        password=get_secret("{{ secrets.password }}").get_secret_value(),
+        {% if secrets.certificate is not none %}cert_path=get_secret("{{ secrets.certificate | safe }}").get_secret_value(){% endif %}
     )
     conn_string = get_secret("{{ secrets.conn_string }}").get_secret_value()
     return couchbase.cluster.Cluster(conn_string, couchbase.options.ClusterOptions(authenticator))

libs/agentc_core/agentc_core/tool/templates/semantic_search.jinja

@@ -28,6 +28,7 @@ secrets:
       conn_string: CB_CONN_STRING
       username: CB_USERNAME
       password: CB_PASSWORD
+      certificate: CB_CERTIFICATE
 
 # Couchbase semantic search tools always involve a vector search.
 vector_search:

libs/agentc_core/agentc_core/tool/templates/sqlpp_query.jinja

@@ -36,5 +36,6 @@ secrets:
         conn_string: CB_CONN_STRING
         username: CB_USERNAME
         password: CB_PASSWORD
+        certificate: CB_CERTIFICATE
 */
 <<< Replace me with your SQL++ query! >>>

libs/agentc_core/agentc_core/util/ddl.py

@@ -310,7 +310,7 @@ def create_vector_index(
 
 def create_gsi_indexes(bucket, cluster, kind, print_progress):
     """Creates required indexes at publish"""
-    progress_bar = tqdm.tqdm(range(4))
+    progress_bar = tqdm.tqdm(range(5))
     progress_bar_it = iter(progress_bar)
 
     completion_status = True
@@ -332,6 +332,22 @@ def create_gsi_indexes(bucket, cluster, kind, print_progress):
         all_errs += err
         completion_status = False
 
+    # Primary index on kind_metadata
+    primary_idx_metadata_name = f"v1_agent_catalog_primary_{kind}_metadata"
+    primary_idx = f"""
+        CREATE PRIMARY INDEX IF NOT EXISTS `{primary_idx_metadata_name}`
+        ON `{bucket}`.`{DEFAULT_CATALOG_SCOPE}`.`{kind}_metadata` USING GSI;
+    """
+    if print_progress:
+        next(progress_bar_it)
+        progress_bar.set_description(primary_idx_name)
+    res, err = execute_query(cluster, primary_idx)
+    for r in res.rows():
+        logger.debug(r)
+    if err is not None:
+        all_errs += err
+        completion_status = False
+
     # Secondary index on catalog_identifier
     cat_idx_name = f"v1_agent_catalog_{kind}cat_version_identifier"
     cat_idx = f"""

libs/agentc_core/tests/tool/resources/sqlpp_query/positive_4.sqlpp

@@ -0,0 +1,26 @@
+/*
+name: tool_4
+
+record_kind: sqlpp_query
+
+description: >
+    i am a dummy tool
+    hello i am a dummy tool
+
+input: >
+    {
+      "type": "object",
+      "properties": {
+        "source_airport": { "type": "string" },
+        "destination_airport": { "type": "string" }
+      }
+    }
+
+secrets:
+    - couchbase:
+        conn_string: CB_CONN_STRING
+        username: CB_USERNAME
+        password: CB_PASSWORD
+        certificate: CB_CERTIFICATE
+*/
+SELECT 1;

templates/agents/with_controlflow/.env.example

@@ -4,8 +4,9 @@ AGENT_CATALOG_USERNAME=Administrator
 AGENT_CATALOG_PASSWORD=password
 AGENT_CATALOG_BUCKET=travel-sample
 
-# In case of capella instance or if secure connection is required
-# replace couchbase with couchbases in AGENT_CATALOG_CONN_STRING and add the following
+# For Capella instances or if a security certificate is required...
+# ...replace 'couchbase' with 'couchbases' in AGENT_CATALOG_CONN_STRING above...
+# ...and modify the line below to point to the root certificate on your local system.
 # AGENT_CATALOG_CONN_ROOT_CERTIFICATE=/path/to/cluster/root/certificate/on/local/system
 
 # Couchbase-specific environment variables (for the travel-agent example tools).

templates/agents/with_controlflow/.gitignore

@@ -5,3 +5,4 @@
 __pycache__
 .env
 poetry.lock
+*.pem

templates/agents/with_controlflow/agent.py

@@ -12,7 +12,6 @@
 import pydantic
 import uuid
 
-from pydantic import SecretStr
 from utils import TaskFactory
 
 # Make sure you populate your .env file with the correct credentials!
@@ -29,9 +28,10 @@
     # The 'values' of this dictionary map to actual values required by the tool.
     # In this case, we get the Couchbase connection string, username, and password from environment variables.
     secrets={
-        "CB_CONN_STRING": SecretStr(os.getenv("CB_CONN_STRING")),
-        "CB_USERNAME": SecretStr(os.getenv("CB_USERNAME")),
-        "CB_PASSWORD": SecretStr(os.getenv("CB_PASSWORD")),
+        "CB_CONN_STRING": os.getenv("CB_CONN_STRING"),
+        "CB_USERNAME": os.getenv("CB_USERNAME"),
+        "CB_PASSWORD": os.getenv("CB_PASSWORD"),
+        "CB_CERTIFICATE": os.getenv("CB_CERTIFICATE"),
     },
 )
 

templates/agents/with_controlflow/tools/find_direct_flights.sqlpp

@@ -57,6 +57,7 @@ secrets:
         conn_string: CB_CONN_STRING
         username: CB_USERNAME
         password: CB_PASSWORD
+        certificate: CB_CERTIFICATE
 */
 
 FROM

templates/agents/with_controlflow/tools/find_one_layover_flights.sqlpp

@@ -58,6 +58,7 @@ secrets:
         conn_string: CB_CONN_STRING
         username: CB_USERNAME
         password: CB_PASSWORD
+        certificate: CB_CERTIFICATE
 */
 
 FROM