Implement example, start building new bootstrap tool

config/vdbes/quickflix.yml

@@ -0,0 +1,50 @@
+schema_name: quickflix
+
+vdbes:
+  - name: order_flow
+    tables:
+      - name: tickets
+        writable: true
+      - name: customers
+        writable: true
+    max_staleness_ms: 0
+    query_interface: SQL_POSTGRESQL
+
+  - name: transform
+    tables:
+      - name: tickets
+      - name: customers
+      - name: sales_history
+        writable: true
+    max_staleness_ms: 3600000  # 1 hour
+    query_interface: SQL_AWS_REDSHIFT
+
+  - name: analysis
+    tables:
+      - name: sales_history
+    max_staleness_ms: 0
+    query_interface: SQL_AWS_REDSHIFT
+
+tables:
+  - name: tickets
+    columns:
+      - name: t_customer_id
+        type: INT_64
+      - name: t_movie_id
+        type: INT_64
+
+  - name: customers
+    columns:
+      - name: c_customer_id
+        type: INT_64
+      - name: c_name
+        type: STRING
+
+  - name: sales_history
+    columns:
+      - name: sh_customer_id
+        type: INT_64
+      - name: sh_movie_id
+        type: INT_64
+      - name: sh_name
+        type: STRING

proto/interface/blueprint.proto

@@ -9,6 +9,7 @@ message Blueprint {
   Provisioning aurora = 101;
   Provisioning redshift = 102;
   RoutingPolicy policy = 201;
+  repeated PhysicalSnapshot snapshots = 301;
 }
 
 message RoutingPolicy {

setup.py

@@ -45,6 +45,7 @@
     "ddsketch",
     "tqdm",
     "psycopg[binary]",
+    "pydantic",
 ]
 
 DEV_REQUIRES = [
@@ -68,7 +69,6 @@
 UI_REQUIRES = [
     "fastapi",
     "uvicorn[standard]",
-    "pydantic",
     "requests",
     "types-requests",
 ]

src/brad/admin/bootstrap_vdbes.py

@@ -0,0 +1,27 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def register_admin_action(subparser) -> None:
+    parser = subparser.add_parser(
+        "bootstrap_vdbes", help="Set up a new virtual infrastructure on BRAD."
+    )
+    parser.add_argument(
+        "--physical-config-file",
+        type=str,
+        required=True,
+        help="Path to BRAD's physical configuration file.",
+    )
+    parser.add_argument(
+        "--vdbe-file",
+        type=str,
+        required=True,
+        help="Path to the virtual infrastructure definition to boostrap.",
+    )
+    parser.set_defaults(admin_action=bootstrap_vdbes)
+
+
+# This method is called by `brad.exec.admin.main`.
+def bootstrap_vdbes(_args) -> None:
+    logger.info("Running VDBE boostrap.")

src/brad/blueprint/vdbe.py

@@ -0,0 +1,25 @@
+import enum
+
+from pydantic import BaseModel
+from typing import Set
+
+# This class contains Python definitions
+
+
+class QueryInterface(enum.Enum):
+    SqlPostgresql = "SQL_POSTGRESQL"
+    SqlMysql = "SQL_MYSQL"
+    SqlAwsRedshift = "AWS_REDSHIFT"
+    SqlAwsAthena = "AWS_ATHENA"
+
+
+class VirtualTable(BaseModel):
+    name: str
+    writable: bool
+
+
+class VirtualEngine(BaseModel):
+    name: str
+    query_interface: QueryInterface
+    max_staleness_ms: int
+    tables: Set[VirtualTable]

src/brad/exec/admin.py

@@ -16,6 +16,7 @@
 import brad.admin.clean_dataset as clean_dataset
 import brad.admin.alter_schema as alter_schema
 import brad.admin.table_adjustments as table_adjustments
+import brad.admin.bootstrap_vdbes as bootstrap_vdbes
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +32,7 @@ def register_command(subparsers) -> None:
         help="Set to enable debug logging.",
     )
     admin_subparsers = parser.add_subparsers(title="Administrative Actions")
+    bootstrap_vdbes.register_admin_action(admin_subparsers)
     bootstrap_schema.register_admin_action(admin_subparsers)
     drop_schema.register_admin_action(admin_subparsers)
     bulk_load.register_admin_action(admin_subparsers)

src/brad/proto_gen/interface/blueprint_pb2.pyi

@@ -18,14 +18,16 @@ ENGINE_REDSHIFT: Engine
 ENGINE_ATHENA: Engine
 
 class Blueprint(_message.Message):
-    __slots__ = ["aurora", "redshift", "policy"]
+    __slots__ = ["aurora", "redshift", "policy", "snapshots"]
     AURORA_FIELD_NUMBER: _ClassVar[int]
     REDSHIFT_FIELD_NUMBER: _ClassVar[int]
     POLICY_FIELD_NUMBER: _ClassVar[int]
+    SNAPSHOTS_FIELD_NUMBER: _ClassVar[int]
     aurora: Provisioning
     redshift: Provisioning
     policy: RoutingPolicy
-    def __init__(self, aurora: _Optional[_Union[Provisioning, _Mapping]] = ..., redshift: _Optional[_Union[Provisioning, _Mapping]] = ..., policy: _Optional[_Union[RoutingPolicy, _Mapping]] = ...) -> None: ...
+    snapshots: _containers.RepeatedCompositeFieldContainer[PhysicalSnapshot]
+    def __init__(self, aurora: _Optional[_Union[Provisioning, _Mapping]] = ..., redshift: _Optional[_Union[Provisioning, _Mapping]] = ..., policy: _Optional[_Union[RoutingPolicy, _Mapping]] = ..., snapshots: _Optional[_Iterable[_Union[PhysicalSnapshot, _Mapping]]] = ...) -> None: ...
 
 class RoutingPolicy(_message.Message):
     __slots__ = ["policy"]