notebook updates

notebooks/aws/aws.ipynb

@@ -97,7 +97,7 @@
    "source": [
     "# get multiple regions asynchronously\n",
     "from pystackql import StackQL\n",
-    "stackql = StackQL(output='pandas')\n",
+    "stackql = StackQL(download_dir='/srv/stackql', output='pandas')\n",
     "stackql.executeStmt(\"REGISTRY PULL aws\")"
    ]
   },
@@ -127,6 +127,16 @@
     "instances_df = await stackql.executeQueriesAsync(queries)\n",
     "instances_df"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%stackql\n",
+    "select PolicyName, Arn from aws.iam.policies where region = '$region'"
+   ]
   }
  ],
  "metadata": {

notebooks/azure/azure.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/github/github.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/google/google.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {
@@ -20,6 +20,20 @@
     "region = 'australia-southeast1'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%stackql\n",
+    "select \n",
+    "split_part(split_part(name, '/', -1), '.', 1) as service, \n",
+    "json_extract(config, '$$.documentation.summary') as summary\n",
+    "from google.serviceusage.services\n",
+    "where parent = '$project' and parentType = 'projects' and filter = 'state:ENABLED'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

notebooks/google/google_audit/1_res_heirarchy.ipynb

@@ -86,6 +86,26 @@
     "## Enabled Services by Project"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get service usage across all projects\n",
+    "serviceusage_queries = [\n",
+    "    f\"\"\"\n",
+    "    select \n",
+    "    '{project}' as project,\n",
+    "    split_part(split_part(name, '/', -1), '.', 1) as service\n",
+    "    from google.serviceusage.services\n",
+    "    where parent = '{project}' and parentType = 'projects' and filter = 'state:ENABLED'\n",
+    "    \"\"\"\n",
+    "    for project in all_projects\n",
+    "]\n",
+    "serviceusage_df = run_stackql_queries(serviceusage_queries)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -94,7 +114,7 @@
    "source": [
     "\n",
     "# service usage summary\n",
-    "plot_serviceusage()"
+    "plot_serviceusage(serviceusage_df)"
    ]
   },
   {

notebooks/google/google_audit/includes/1-res-heirarchy-setup.ipynb

@@ -145,41 +145,24 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def plot_serviceusage():\n",
-    "    all_services = [\n",
-    "        ('servicenetworking', servicenetworking_projects),\n",
-    "        ('compute', compute_projects),\n",
-    "        ('storage', storage_projects),\n",
-    "        ('composer', composer_projects),\n",
-    "        ('bigquery', bigquery_projects),\n",
-    "        ('logging', logging_projects),\n",
-    "        ('dialogFlow', dialogflow_projects),\n",
-    "        ('cloudfunctions', cloudfunctions_projects),\n",
-    "        ('contactcenteraiplatform', contactcenteraiplatform_projects),\n",
-    "        ('monitoring', monitoring_projects),\n",
-    "        ('datastudio', datastudio_projects),\n",
-    "        ('notebooks', notebooks_projects),\n",
-    "        ('aiplatform', aiplatform_projects),\n",
-    "        ('artifactregistry', artifactregistry_projects),\n",
-    "        ('containerregistry', containerregistry_projects),\n",
-    "    ]\n",
+    "def plot_serviceusage(serviceusage_df):\n",
+    "    # Group by 'service' and count the number of projects for each service\n",
+    "    service_counts = serviceusage_df.groupby('service')['project'].count().reset_index()\n",
     "    \n",
-    "    # Count the number of projects for each service\n",
-    "    service_counts = [(service[0], len(service[1])) for service in all_services]\n",
+    "    # Sort the data frame by counts in descending order\n",
+    "    service_counts = service_counts.sort_values(by='project', ascending=False)\n",
     "    \n",
-    "    # Sort by the number of projects, most to least\n",
-    "    service_counts.sort(key=lambda x: x[1], reverse=True)\n",
-    "    \n",
-    "    # Extract sorted service names and counts\n",
-    "    sorted_service_names = [x[0] for x in service_counts]\n",
-    "    sorted_project_counts = [x[1] for x in service_counts]\n",
+    "    # Extract service names and counts\n",
+    "    sorted_service_names = service_counts['service'].tolist()\n",
+    "    sorted_project_counts = service_counts['project'].tolist()\n",
     "    \n",
     "    # Create the bar chart\n",
     "    fig = go.Figure(data=[go.Bar(x=sorted_service_names, y=sorted_project_counts)])\n",
     "    \n",
     "    # Add titles and labels\n",
     "    fig.update_layout(title='Enabled GCP Services',\n",
     "                      xaxis_title='GCP Service',\n",
+    "                      xaxis_tickangle=-45,\n",
     "                      yaxis_title='Number of Projects')\n",
     "    \n",
     "    # Show the figure\n",

notebooks/google/google_audit/includes/shared-setup.ipynb

@@ -20,7 +20,7 @@
     "import plotly.express as px\n",
     "import networkx as nx\n",
     "\n",
-    "conn = psycopg2.connect(\"dbname=stackql user=stackql host=localhost port=5444\")"
+    "conn = psycopg2.connect(\"dbname=stackql user=stackql host=localhost port=5466\")"
    ]
   },
   {

notebooks/k8s/k8s.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/netlify/netlify.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/okta/okta.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/stackql.ipynb

@@ -19,7 +19,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {

notebooks/sumologic/sumologic.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext pystackql"
+    "%load_ext pystackql.magics"
    ]
   },
   {