add k8s migration tests (not in CI)

notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/live/schema.py

@@ -9,7 +9,7 @@
 from ..rate_limiter import is_within_rate_limit
 
 
-def make_schema(settings: dict, worker_pool_name: str) -> Callable:
+def make_schema(settings: dict, worker_pool: str) -> Callable:
     updated_settings = {
         "calls_per_min": 5,
         "rate_limiter_enabled": True,
@@ -28,7 +28,7 @@ def make_schema(settings: dict, worker_pool_name: str) -> Callable:
         helper_functions=[
             is_within_rate_limit
         ],  # Adds ratelimit as this is also a method available to data scientists
-        worker_pool_name=worker_pool_name,
+        worker_pool=worker_pool,
     )
     def live_schema(
         context,

notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/mock/schema.py

@@ -9,7 +9,7 @@
 from .data import schema_dict
 
 
-def make_schema(settings, worker_pool_name) -> Callable:
+def make_schema(settings, worker_pool) -> Callable:
     updated_settings = {
         "calls_per_min": 5,
         "rate_limiter_enabled": True,
@@ -21,7 +21,7 @@ def make_schema(settings, worker_pool_name) -> Callable:
         description="This endpoint allows for visualising the metadata of tables available in BigQuery.",
         settings=updated_settings,
         helper_functions=[is_within_rate_limit],
-        worker_pool_name=worker_pool_name,
+        worker_pool=worker_pool,
     )
     def mock_schema(
         context,

notebooks/scenarios/bigquery/upgradability/0.9.1_helpers/apis/submit_query.py

@@ -2,13 +2,13 @@
 import syft as sy
 
 
-def make_submit_query(settings, worker_pool_name):
-    updated_settings = {"user_code_worker": worker_pool_name} | settings
+def make_submit_query(settings, worker_pool):
+    updated_settings = {"user_code_worker": worker_pool} | settings
 
     @sy.api_endpoint(
         path="bigquery.submit_query",
         description="API endpoint that allows you to submit SQL queries to run on the private data.",
-        worker_pool_name=worker_pool_name,
+        worker_pool=worker_pool,
         settings=updated_settings,
     )
     def submit_query(

notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/011-users-emails-passwords.ipynb

@@ -657,11 +657,6 @@
   }
  ],
  "metadata": {
-  "kernelspec": {
-   "display_name": "syft_3.12",
-   "language": "python",
-   "name": "python3"
-  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
@@ -672,7 +667,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
+   "version": "3.12.5"
   }
  },
  "nbformat": 4,

notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/020-configure-api.ipynb

@@ -216,7 +216,7 @@
     "    description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n",
     "    private_function=private_func,\n",
     "    mock_function=mock_func,\n",
-    "    worker_pool_name=this_worker_pool_name,\n",
+    "    worker_pool=this_worker_pool_name,\n",
     ")\n",
     "\n",
     "high_client.custom_api.add(endpoint=new_endpoint)"
@@ -407,7 +407,7 @@
     "    settings={\n",
     "        \"calls_per_min\": 5,\n",
     "    },\n",
-    "    worker_pool_name=this_worker_pool_name,\n",
+    "    worker_pool=this_worker_pool_name,\n",
     ")"
    ]
   },
@@ -462,7 +462,7 @@
    "outputs": [],
    "source": [
     "submit_query_function = make_submit_query(\n",
-    "    settings={}, worker_pool_name=this_worker_pool_name\n",
+    "    settings={}, worker_pool=this_worker_pool_name\n",
     ")"
    ]
   },
@@ -640,11 +640,6 @@
   }
  ],
  "metadata": {
-  "kernelspec": {
-   "display_name": "syft_3.12",
-   "language": "python",
-   "name": "python3"
-  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",

notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/jobs_k8s.json

@@ -0,0 +1,180 @@
+{
+  "[email protected]": [
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_61dd0e",
+      "query": "SELECT table_id, AVG(colname) AS average_score\n    FROM data_10gb.subreddits\n    GROUP BY table_id\n    LIMIT 713078",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "data_10gb",
+        "table": "subreddits",
+        "groupby_col": "table_id",
+        "score_col": "colname",
+        "limit": 713078
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_61dd0e",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    },
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_7e0b14",
+      "query": "SELECT subreddit_id, AVG(score) AS average_score\n    FROM data_10gb.comments\n    GROUP BY subreddit_id\n    LIMIT 770329",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "data_10gb",
+        "table": "comments",
+        "groupby_col": "subreddit_id",
+        "score_col": "score",
+        "limit": 770329
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_7e0b14",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    }
+  ],
+  "[email protected]": [
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_6e409e",
+      "query": "SELECT table_id, AVG(colname) AS average_score\n    FROM test_1gb.subreddits\n    GROUP BY table_id\n    LIMIT 969621",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "test_1gb",
+        "table": "subreddits",
+        "groupby_col": "table_id",
+        "score_col": "colname",
+        "limit": 969621
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_6e409e",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    },
+    {
+      "user_email": "[email protected]",
+      "func_name": "wrong_syntax_query_201be7",
+      "query": "SELECT * FROM table INCORRECT SYNTAX",
+      "job_type": "wrong_syntax_query",
+      "settings": {},
+      "should_succeed": false,
+      "should_submit": true,
+      "code_path": "wrong_syntax_query_201be7",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    }
+  ],
+  "[email protected]": [
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_cac747",
+      "query": "SELECT table_id, AVG(colname) AS average_score\n    FROM data_10gb.subreddits\n    GROUP BY table_id\n    LIMIT 865657",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "data_10gb",
+        "table": "subreddits",
+        "groupby_col": "table_id",
+        "score_col": "colname",
+        "limit": 865657
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_cac747",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    },
+    {
+      "user_email": "[email protected]",
+      "func_name": "job_too_much_text_7d0b7e",
+      "query": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "job_type": "job_too_much_text",
+      "settings": {},
+      "should_succeed": false,
+      "should_submit": true,
+      "code_path": "job_too_much_text_7d0b7e",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    }
+  ],
+  "[email protected]": [
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_9ab2f5",
+      "query": "SELECT table_id, AVG(colname) AS average_score\n    FROM test_1gb.subreddits\n    GROUP BY table_id\n    LIMIT 897138",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "test_1gb",
+        "table": "subreddits",
+        "groupby_col": "table_id",
+        "score_col": "colname",
+        "limit": 897138
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_9ab2f5",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    },
+    {
+      "user_email": "[email protected]",
+      "func_name": "job_long_name_d6277aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "query": "SELECT subreddit_id, AVG(score) AS average_score\n    FROM data_10gb.comments\n    GROUP BY subreddit_id\n    LIMIT 591345",
+      "job_type": "job_long_name",
+      "settings": {
+        "dataset": "data_10gb",
+        "table": "comments",
+        "groupby_col": "subreddit_id",
+        "score_col": "score",
+        "limit": 591345
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "job_long_name_d6277aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    }
+  ],
+  "[email protected]": [
+    {
+      "user_email": "[email protected]",
+      "func_name": "simple_query_edeb0b",
+      "query": "SELECT subreddit_id, AVG(score) AS average_score\n    FROM data_10gb.comments\n    GROUP BY subreddit_id\n    LIMIT 133654",
+      "job_type": "simple_query",
+      "settings": {
+        "dataset": "data_10gb",
+        "table": "comments",
+        "groupby_col": "subreddit_id",
+        "score_col": "score",
+        "limit": 133654
+      },
+      "should_succeed": true,
+      "should_submit": true,
+      "code_path": "simple_query_edeb0b",
+      "admin_reviewed": true,
+      "result_as_expected": true
+    },
+    {
+      "user_email": "[email protected]",
+      "func_name": "job_funcname_xss_338b43<script>alert('XSS in funcname')</script>",
+      "query": "SELECT table_id, AVG(colname) AS average_score\n    FROM test_1gb.subreddits\n    GROUP BY table_id\n    LIMIT 64464",
+      "job_type": "job_funcname_xss",
+      "settings": {
+        "dataset": "test_1gb",
+        "table": "subreddits",
+        "groupby_col": "table_id",
+        "score_col": "colname",
+        "limit": 64464
+      },
+      "should_succeed": true,
+      "should_submit": false,
+      "code_path": null,
+      "admin_reviewed": false,
+      "result_as_expected": null
+    }
+  ]
+}

notebooks/scenarios/bigquery/upgradability/0.9.1_notebooks/users_k8s.json

@@ -0,0 +1,52 @@
+[
+  {
+    "name": "Richard Porter",
+    "email": "[email protected]",
+    "password": "Fmo4!MxcZ(",
+    "role": "ServiceRole.DATA_SCIENTIST",
+    "new_password": null,
+    "email_disabled": true,
+    "reset_password": false,
+    "reset_token": null
+  },
+  {
+    "name": "Christine Peterson",
+    "email": "[email protected]",
+    "password": "4J13O(x&!G",
+    "role": "ServiceRole.DATA_SCIENTIST",
+    "new_password": null,
+    "email_disabled": false,
+    "reset_password": true,
+    "reset_token": "DgSgUZ2gOpvm"
+  },
+  {
+    "name": "Jon Wilson",
+    "email": "[email protected]",
+    "password": "h5TEQw#t%4",
+    "role": "ServiceRole.DATA_SCIENTIST",
+    "new_password": null,
+    "email_disabled": false,
+    "reset_password": false,
+    "reset_token": null
+  },
+  {
+    "name": "Tyrone Flores",
+    "email": "[email protected]",
+    "password": "&#CJFvxb3_",
+    "role": "ServiceRole.DATA_SCIENTIST",
+    "new_password": null,
+    "email_disabled": false,
+    "reset_password": false,
+    "reset_token": null
+  },
+  {
+    "name": "Brian Bradford",
+    "email": "[email protected]",
+    "password": "crZ#4Heh6e",
+    "role": "ServiceRole.DATA_SCIENTIST",
+    "new_password": null,
+    "email_disabled": false,
+    "reset_password": false,
+    "reset_token": null
+  }
+]