Merge pull request #2197 from benipeled/packer

Add cloud-image build scripts
scylladb · Mar 5, 2024 · add40bb · add40bb
2 parents 842d9da + 062edf0
commit add40bb
Show file tree

Hide file tree

Showing 32 changed files with 2,889 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,4 @@ docs/.Python
 docs/poetry.lock
 grafana/build/
 CURRENT_VERSION.sh
-
+packer-manifest.json
diff --git a/packer/README.md b/packer/README.md
@@ -0,0 +1,58 @@
+# Scylla Monitor Image
+
+## Requirements
+- Packer >= v1.10.0
+- Packer AWS and GCP plugins
+
+To install the required plugins, run the following:
+
+```shell
+packer plugins install github.com/hashicorp/googlecompute
+packer plugins install github.com/hashicorp/amazon
+```
+
+## Build
+To build the Scylla Monitor Image, make sure you have [Authentication](#authentication) set up, and run the following command from the `siren-devops/cluster/monitor` directory:
+
+```shell
+packer build -var monitor_version="4.6.1" scylla-monitor-template.json
+```
+You can build a specific cloud only by using the `-only` flag. for example:
+```shell
+# AWS only
+packer build -only=amazon-ebs -var monitor_version="4.6.1" scylla-monitor-template.json
+
+# GCP only
+packer build -only=googlecompute -var monitor_version="4.6.1" scylla-monitor-template.json
+```
+## Variables
+
+
+The Scylla Monitor Image uses default variables that are declared in the packer template file, for example `aws_source_ami`, `gcp_project_id` etc.  
+You can override these default variables by creating a `variables.json` with the desired variable values, for example:
+
+```json
+{
+  "monitor_version": "4.6.1",
+  "aws_subnet_id": "your_aws_subnet_id",
+  "gcp_project_id": "your_gcp_project_id",
+  "gcp_zone": "your_gcp_zone"
+}
+```
+And when running the packer build command, include the `-var-file` option to specify the `variables.json` file:
+
+```shell
+packer build -var-file=variables.json scylla-monitor-template.json
+```
+
+
+## Authentication
+
+### AWS
+Ensure `aws_access_key_id` and `aws_secret_access_key` are configured either in a local credentials file (ex. `~/.aws/credentials`) or as environment variables.
+
+#### GCP
+Set your GCP service account key as an environment variable:
+```shell
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-file.json"
+```
diff --git a/packer/files/UA.sh b/packer/files/UA.sh
@@ -0,0 +1 @@
+UA_ANALTYICS=UA-43975320-4
diff --git a/packer/files/add_centos_user.sh b/packer/files/add_centos_user.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/bash
+print_usage() {
+    echo "add_centos_user.sh [--os centos/ubuntu] [--copy]"
+    exit 1
+}
+OS="centos"
+COPY=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        "--os")
+            OS="$2"
+            shift 2
+            ;;
+        "--copy")
+            COPY="1"
+            shift 1
+            ;;
+        *)
+            print_usage
+            ;;
+    esac
+done
+
+if [ "$OS" = "ubuntu" ]; then
+    sudo useradd -m -s $(which bash) -G sudo -G docker centos
+    echo "centos ALL=(ALL) NOPASSWD:ALL" |sudo tee -a  /etc/sudoers.d/100-cloud-cntos-user > /dev/null
+    if [ "$COPY" = "1" ]; then
+        sudo cp -r  /home/ubuntu/scylla-grafana-monitoring-scylla-monitoring /home/centos/
+    fi
+    sudo chown -R centos:centos /home/centos
+fi
diff --git a/packer/files/clear_files.sh b/packer/files/clear_files.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+BASE=`dirname "$(readlink -f "$0")"`
+echo "clearing $BASE/scylla-grafana-monitoring-scylla-monitoring/grafana/build/*/$1.*.json"
+for FILE in $BASE/scylla-grafana-monitoring-scylla-monitoring/grafana/build/ver_*/$1.*.json; do
+  rm -f "$FILE"
+done
diff --git a/packer/files/clusters-overview.dbaas.template.json b/packer/files/clusters-overview.dbaas.template.json
@@ -0,0 +1,193 @@
+{
+    "dashboard": {
+        "class": "dashboard",
+        "uid": "dbaas",
+        "originalTitle": "Scylla Cloud Solution",
+        "rows": [
+            {
+                "collapse": false,
+                "editable": true,
+                "height": "200px",
+                        "panels": [
+                            {
+                                "class": "text_panel",
+                                "content": "<img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAkYAAAB2CAYAAAAtOFzoAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAD6VJREFUeNrs3Y1V20oWwHGRkwLcwSoVxKkgooJHKoioAKggpgKTCuxUYKcCOxXAqwC/CvBWwOqGcY4fi0HSvZov/3/n6JDzdjHyaEZz57soAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABA5E6O5Ys+Pj6Omx+jgwlxcrImO/RO27L5Ub7yf7lr0ndLSgEACIzCVNRV80Ouz831akD0AgmQ7prrb/l3U6FvyCb/Ci7l+uh+dk3bu2dpe0eqAgAIjIaptM+aH38111nHyrpNZf6zuebHGCTtpasEmqXxx0t6LpvrR0xBUvOda6Pvumm+19zzvU8s8nxz38uU0r2538kRlUlJozqVPJlxw7uLG3rN4SuDjprrsrnuH/1YuUKR/Yu3uWbN9fDoz62rGKN48Rl+r7HPoCi1e352/6u+N3xsFbPmHUbNoU7/PvXNJSkHL61LjwHRSwHSOMM0rTSVk5H7GIJPFxia5BWPwaxFIDsNmOYERgRGuab9PamHoXuJQlfewSuRASrVWNJ0ZyHPOnA+s+oxO/NwvwujoDRkmhMYERjFnvaaclaTgml4l1imlF4aibyrSG7p0g0BjRMu6JPI0nRHgolgvUduPsC50cdNB36GlUsvrXPmQQCHG5DKcvaVVCQwss6UEm3fFrYTqy1IULRyhSa1XiJJz28R3+bIpW2QlpabgGwxCbk0mhR9yMzgM27YsgJ41YXy96scp2AQGIUNimYR3+JdSivW3NDOrQvqUjAL2A0tvUYWvSgXQwxTuYBLG5RL3r3mdQgcLGdSdi3eQRekJoGRRYaUyjvmuTyyzPxLQgVcCveiiK/nLcrgyHBITdL7m/GzLI1etAyhAa+rjd6Zdch5fMggMHIZKOZKXIKi01QqlQR63toER97zguGQ2qXxkOvUoGwwhAa8zbKnh6X7BEYq0sIuI723LUGR9/T+FDC9rYbUZkbPsyr0E67l+zCEBrxe1s6M6yGG0wiMemfGMuLImqDIL+mZ+xByd2z3rK8MPqoyWmln8TwZQgP8BzIjlu4TGPUV62qpXVCUxDlfCczRess8liDUHaWwDh3UGE24XoY49gNIiWugVwN8NL1GEXsfcWa0jqjXrufhv8/+e9eDZr8kFBT5mKO1Oxj2nxf+t8+uAu9biV81aX0TWbLKkJp22whZvn/Z57sZTbi23KMJyNlQDfSx9Bwzvw9dXv6XhsdLtFoFID0rLc4HqxNLx+mAu1KftZ0I7fZMmnQ4wuXBx27RgfPnQ5+J5EY7XJ9Fmq7sfN0undj52lPDcuDzImekMry8IPdMFIVhmkFQVA1RkLWrqlyg+vDGYbJj8uhgz3SRY5oSGBEYRdxAf01JSqNthtSqjV4+D6meieYCDCumwcor592tUtnjw/DQ1rLD39QemvwQc/oSGBEYRZbOPg4pn5LSaJMZx7F0T7p7STEoqo17iUYD3ecs5ReEUYty0fJvTWJoMBAYERgdSRrXj348PLLhIwYu9I+cRWPa0pl5uNdpyktXjYbUKg+9U6uc05J3JIFRhOU6iwYL4siQqpYx6Wc2t4iJgX56OB/dYb6v/Y2FQau0JDCijBMYeSvTnRYJkepxeUcSZMdif4z1yckJy7lbcFs3aHePHh9qNRrtcH2d0iHHQAbv0C5Ko01fQWB0MNovj/VhurFqi2MivlA0OgVHk+JpLyeNbwfmGmh77tYR7gUFxPwOrfuWNU35J/UJjIZUH/HztNif5ppjInrR9rBJQH/57CU9KXQ7XLORI9BN32OoNsqyVrF0n8DorZe5xsURT8D+S/n7G3oX+jEaUrvYvRyNdrhmCA3o5mvP3/vhyprmmB16jQiMDtIOSUhX6OpIgyP1XBSKhCo4mijz72jv5TgtdMeOMIQGdODm+ZU9f32+C5A072+W7hMYDRUY7SqYW7fCbXQkhbpSfsTGHZIKHe3QVe2G0LRB7hWPAuikb2/Rctcz6w5m3ijqrZrHQGD0Uqt7q8hYz0nr+94FSGXmz1LbQ8ZJ6zb5VwL7G4N8q3GdykHHQEQNy76Nyx+G79ILngaBkY9KerQXIN263YpzDJI+Kn//B8XBzLVhcN/VnRvSA9Be396iresl2vddcR8lGz4SGB3yfaDPlV6VqQuS7t2uy1Umz1IT7G3pYbDjej1DrQZjFRrQgWso9w1G5i+Uf2kUrQMEacg5MHIZa+4hkJClmSu3M7Cc23WW8LPUDKURFNnnYXkx+p78zBAa0F2t+N1DjXhND3zF0VYERgdf8oV+6X5bu0lvi70gqUrsWWommf+iKAyWhzee/hZDaEA/fef1rA9th+EWsmwD3BNyDoxchguxsmYXJK3cnKQ69odosPJuQ1EYJA/7HFJjCA3o/u6sFY3Kt3qF5opbq1m6T2BUvBJ1h9yLRbozZ24+UhVxUmm7XQmMhsvDaw95+IYhNKCXvj0z2xbbm2jnyl7yeAiMDlUsV4GDI1EWTz1IM6J49DDksPCmYGNOoDPX2O3bqHxz5bTBJGyG0wiM3gyOzgt/c44OqV2AVJJ10CH/Djmkds7ZdkAvmtVfbXuDNJOwRyzdJzB6q3KZNz8+KSNwC9LCuGXVADrm32Vhv4nmjRuqA9CBcon+XduhayZhExj5qFzk2IrT5p+ngQMkGU5bMKyGjix7PTcFQ2hAX7Xid7vOHZprGuIZ7bVHYDRwgLTeC5BCHWUhLY5ZJEmyMfguGD7fWg6pMYQG9OAatL0nXfeoc7SnCrDhI4FR5wDpS/PPD8XT0v6N51s4i2FTyEN7aRAYRZlnLYbUGEIDFO/tov8S/WXXBokbdtOsGq2Z10pg1CswaC6pLCRAknlIN4W/3ZynGSThR4qCV9peI4bQgP40hzT37f3RLt1nrhGBkSpIkolxV80lAdKuJ2nIIKmMZAxY04PARHK/eXQb8veBY+Xe1WXPX98oemqll1hTbtnwkcDIrALa9STtB0mbAf5UDGPAmkJX0lUL4Ahoel569/q4xoxmCF2CojMeH4HRUEGSBEjWq9qqCL7i38rfp9AByJZr/Gnec9q5gdrhtG88RQKjIYOk/VVtFsNsZQTdnNpAj5UPAHKm6S1aahe5GEzCLlm6T2DkK0CSYba5wceFnqejDfDYLwNAllzDtVZ8xE+jW6HXiMAomQBJVgktE/8OW4PgiEIHIEeaJfptDoxtSzsJu2I+KIGRT9pdiasIvoN2I7GKXiMAGdI0+swazQaTsGnAEhj5Y5RhQ7O4/ym5AUAu3Ca8peIjvhvfkvbzzlJcui8H4jbXrLnuH/+fHM4+iaU3jMDo3/5JPLjbFDZzjaaeC0xJ1gMwEM2k603bA2M7vKe1k7AlKLpMKCC6bK6H4ukIrfpAkFoVTz1h9y5IClonJBcYyan2kUbLd5Hch0XrRjJy7el5Vq4w0FMFYIhGVxX4fTrE535NIO1HzXVbPI1CdKmz5Xnd+qqDkg+MJChqfqzcNYT/KH43it2I3SRBi3uZDX0OnHuei71g7Nb9NwCwoJ2PM9T0Cu0k7DJk4NDy3X5f9F+tPXJ1UJCD2pMJjFzkv3IJNnbdbSPjz9dktJiOabBq5SyGKnx7Qe7+M/z932Iu8ACSqTO0u0WvDQ7oPtSAtZjT+jXidF8U/VcB7qupD15JaNeb8Nyt1VikC7R6izC97h/tTI3vT1a/PbzxN2c5nw2US14zTpMVadK6/PS1OqJ0ulS+9+qB729s8G4eR5jui0d7jCS0DIp2pIKdKD9/pXxoqwjTrTbOmPfapfwuracd/ma2Q2sERgRGBEaDp5Omcfjgo2FmUPfMMsqb0eTbkxRelkW7yXPS5SlDSPM2p4+7TC8z+y8Muvyu5By2hNOui3XxtF/Ssu0p73vDlH3SeuvSd55bYNS70DZyDYwU+XXt8VavrFcqda18iv7zLC02gm3rrkmnq0BpJENoC8VHzN3Gv4M3YIun1VoaH4Ya8oukztk5ldMqjj4wctFw3fMl+cu9AJ5X3vLQPha2h6ZGkzFfCEhuC5ux3q7pXLp0lvS26PWZuwppW2SAwMj7SzXJF/QAgZFPuzMpU8xLn3wFv24pu+Ydfd3c6ySS+uZ+wD/hJVgV7zMMinbBT+Wx8G9iTEO5ryYdz5Utp1jSWfKCjMmfh2ytA4g+wB4r30sbz+8YafRp9iWSnvhJBElfJf75f7yLNGNfFroVYj5dx3xzTQGXlQ83RR5YtQagTaCg8d3z/Wr/3iiSd+LHgT+/PNrAyD3gVDb7W4fsUu8QHF25VkkOsl2pBkBdf4wMGtVe35VuxEFbj1xEkPxjD8+3OrrAyGgimk9XqdyoG5vNITia5zYRG4AZ7VEZ80DzGLUHgI85ADzDwMiNC6d0LMRVanNdMgiO7nxNvgOQJO2mhz9C3LTRiQUXPP7MAqPiqRsulWGSeYzL8zsERyneuwShpxRZAAca13Whm4eyCTw1QjvX6Czw4au/PNRfXp5PNIGRi5hPi7iO1jgUFCXda+HmHH1JIK3/pHnxtEQ6i6X6AAah7S36Hvj+5wafEbLXaOOhcexFVHOMXDT4qYjnpPrsgqK9tF66tF5HfquyR8c5QRGAQwyW6FsFJpp3sgQW2vPT6oBHKS0H/nxvdVV0q9IkczSXVNgxLYOXSvk8t/ktLq2ll+7cQ7Tfp/VxGsPGZQCip+0pWUbS+NLOcdIenKupT7YDB5feevTexZrLXYX4wUMU2iZK/ZTzSij33XbBaOiXw9bdx6cUtkIAEJbREv0fMXwX15OvbaR+C/gVhurQmPvcSPldzBne9WjIXJjTAAGSVMrSY3Ea687W1tH+XjAqPUi+hzN3AZEcrzJh6AxAS9ol+hsXkMRCG6SV7qy4IHV2Yb+45/d5mT6/x/sUcr3rOVjvHUYqk+zKgSpnKSDfj/XYib3u0Lkbt5e0PiuG23VU0lm6SJdHFgytqc9ezAspiKFXdc3z/OOjMj1+RpZu8v79rPyMcRFotEUW97g9law2fPQ+xzTZwyj3Jtt9dj/7TjiTAiXLDNcM3bya3uVeeo8VmX7zLM03pC4AZFVfSH28MgiOzkNMYznJ7EHs9kIav1Ex/76olE2CU0nvsni9R0lajlsCTwA4quBI5jv1GeqUuvkLB4YDAIDcAqSqueTw7zYemmsScMuB3054bAAAYOAAqSye5qvKdIzngY/0DP2KbBI8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAR+p8AAwBZhVVEP80/TAAAAABJRU5ErkJggg==\" height=\"70\">\n<span style=\"font-size:40px\"> DBAAS Overview</span><hr style=\"border-top: 3px solid #5780c1;\">",
+                                "gridPos": {
+                                    "x": 0,
+                                    "y": 0,
+                                    "w": 24,
+                                    "h": 4
+                                  }
+                            }
+                        ],
+                "title": "New row"
+            },
+            {
+                "class": "row",
+                "height": "150px",
+                "panels": [
+                    {
+                        "class": "single_stat_panel",
+                        "targets": [
+                            {
+                                "expr": "count(up{job=\"monitoring\"})",
+                                "intervalFactor": 1,
+                                "legendFormat": "Total number of Clusters",
+                                "refId": "A",
+                                "step": 40
+                            }
+                        ],
+                        "title": "Clusters"
+                    },
+                    {
+                        "class": "single_stat_panel_fail",
+                        "targets": [
+                            {
+                                "expr": "count(scrape_samples_scraped{job=\"monitoring\"}==0) OR vector(0)",
+                                "intervalFactor": 1,
+                                "legendFormat": "Unreachable Prometheus Servers",
+                                "refId": "A",
+                                "step": 20
+                            }
+                        ],
+                        "thresholds": "1,2",
+                        "title": "Prometheus"
+                    },
+                    {
+                        "class": "single_stat_panel_fail",
+                        "targets": [
+                            {
+                                "expr": "count(scrape_samples_scraped{job=\"alarms\"}==0) OR vector(0)",
+                                "intervalFactor": 1,
+                                "legendFormat": "Unreachable Alarm manager",
+                                "refId": "A",
+                                "step": 20
+                            }
+                        ],
+                        "thresholds": "1,2",
+                        "title": "Alarm-Manager"
+                    },
+                    {
+                        "class": "single_stat_panel_fail",
+                        "targets": [
+                            {
+                                "expr": "count(scrape_samples_scraped{job=\"node_exporter\"}==0) OR vector(0)",
+                                "intervalFactor": 1,
+                                "legendFormat": "Unreachable Node Exporter",
+                                "refId": "A",
+                                "step": 20
+                            }
+                        ],
+                        "thresholds": "1,2",
+                        "title": "node-exporter"
+                    },
+                    {
+                        "class": "text_panel",
+                        "content": "##  ",
+                        "mode": "markdown",
+                        "span": 1,
+                        "style": {}
+                    },
+                    {
+                        "class": "alert_table",
+                        "span": 6,
+                        "title": "Active Alerts"
+                    }
+                ],
+                "title": "Statistic Row"
+            },
+            {
+                "class": "row",
+                "panels": [
+                   {
+                        "class": "percent_panel",
+                        "span": 3,
+                        "targets": [
+                            {
+                                "expr": "100*(sum(node_filesystem_size{cluster=~\"$cluster|$^\", mountpoint=~\"$mount_point\"}) by (cluster,instance, mountpoint)-sum(node_filesystem_avail{cluster=~\"$cluster|$^\", mountpoint=~\"$mount_point\"}) by (cluster,instance, mountpoint))/sum(node_filesystem_size{cluster=~\"$cluster|$^\", mountpoint=~\"$mount_point\"}) by (cluster,instance, mountpoint)",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Disk Size"
+                    },
+                    {
+                        "class": "percent_panel",
+                        "span": 3,
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_cpu_seconds_total{cluster=~\"$cluster|$^\", mode=\"idle\"}[30s])) by (cluster,instance)",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "CPU Load"
+                    },
+                    {
+                        "class": "percent_panel",
+                        "span": 3,
+                        "targets": [
+                            {
+                                "expr": "100*(sum(node_memory_MemTotal_bytes{cluster=~\"$cluster|$^\"}) by (cluster,instance) - sum(node_memory_MemFree_bytes{cluster=~\"$cluster|$^\"}) by (cluster,instance))/sum(node_memory_MemTotal_bytes{cluster=~\"$cluster|$^\"}) by (cluster,instance)",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Memory Usage"
+                    },
+                    {
+                        "class": "bytes_panel",
+                        "span": 3,
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_network_transmit_bytes{cluster=~\"$cluster|$^\", device!~\"lo\"}[30s])) by (cluster,instance)",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Data transfer"
+                    }
+
+                ]
+            }
+        ],
+        "templating": {
+            "list": [
+                {
+                    "class": "template_variable_all",
+                    "label": "cluster",
+                    "name": "cluster",
+                    "query": "label_values(node_filesystem_avail, cluster)"
+                },
+                {
+                    "class": "template_variable_all",
+                    "label": "mountpoint",
+                    "name": "mount_point",
+                    "query": "label_values(node_filesystem_avail,mountpoint)",
+                    "sort": 3
+                }
+            ]
+        },
+		"tags": [
+			"dbaas"
+		],
+        "time": {
+            "from": "now-30m",
+            "to": "now"
+        },
+        "title": "DBaS monitoring",
+        "version": 3
+    }
+}