feat: add support to export logs

server/docker-compose.yml

@@ -1,8 +1,10 @@
 version: '3'
 
 services:
+
   nginx:
     image: nginx:latest
+    restart: always
     container_name: sarthi_nginx
     ports:
       - "80:80"
@@ -13,6 +15,7 @@ services:
 
   sarthi:
     build: .
+    restart: always
     ports:
       - 5000:5000
     volumes:
@@ -29,3 +32,36 @@ services:
       NGINX_PROXY_CONF_LOCATION: ${NGINX_PROXY_CONF_LOCATION}
       ENV: ${ENV:-local}
       DOMAIN_NAME: ${DOMAIN_NAME:-localhost}
+
+  loki:
+    image: grafana/loki:latest
+    restart: always
+    ports:
+      - 3100:3100
+    volumes:
+      - ./logging-config/loki:/etc/loki
+    command: -config.file=/etc/loki/loki-config.yaml
+    depends_on:
+      - promtail
+
+  promtail:
+    image: grafana/promtail:latest
+    restart: always
+    volumes:
+      - /var/log:/var/log
+      - ./logging-config/promtail:/etc/promtail 
+    command: -config.file=/etc/promtail/promtail-config.yaml
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana-1:/var/lib/grafana
+      - ./logging-config/grafana/dashboards:/etc/grafana/provisioning/dashboards
+      - ./logging-config/grafana/datasources:/etc/grafana/provisioning/datasources
+
+volumes:
+  grafana:
+  grafana-1:

server/logging-config/grafana/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'Loki'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    editable: true
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

server/logging-config/grafana/dashboards/docker_logs.json

@@ -0,0 +1,127 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 1,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "dedupStrategy": "none",
+          "enableLogDetails": true,
+          "prettifyLogMessage": true,
+          "showCommonLabels": true,
+          "showLabels": true,
+          "showTime": true,
+          "sortOrder": "Descending",
+          "wrapLogMessage": true
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "editorMode": "builder",
+            "expr": "{compose_service=\"$service_name\"} |~ `(?i)$search_text`",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Logs",
+        "type": "logs"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 39,
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "grafana",
+            "value": "grafana"
+          },
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": true,
+          "multi": false,
+          "name": "service_name",
+          "options": [],
+          "query": {
+            "label": "compose_service",
+            "refId": "LokiVariableQueryEditor-VariableQuery",
+            "stream": "",
+            "type": 1
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "",
+            "value": ""
+          },
+          "hide": 0,
+          "name": "search_text",
+          "options": [
+            {
+              "selected": true,
+              "text": "",
+              "value": ""
+            }
+          ],
+          "query": "",
+          "skipUrlSync": false,
+          "type": "textbox"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-5m",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Service logs",
+    "uid": "c9398592-6dc7-48c7-9d19-fba0afa195d0",
+    "version": 4,
+    "weekStart": ""
+  }

server/logging-config/grafana/datasources/datasources.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: true
+    editable: true

server/logging-config/loki/loki-config.yaml

@@ -0,0 +1,68 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+ingester:
+  wal:
+    enabled: true
+    dir: /tmp/wal
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /tmp/loki/boltdb-shipper-active
+    cache_location: /tmp/loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
+  filesystem:
+    directory: /tmp/loki/chunks
+
+compactor:
+  working_directory: /tmp/loki/boltdb-shipper-compactor
+  shared_store: filesystem
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /tmp/loki/rules
+  rule_path: /tmp/loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true

server/logging-config/promtail/promtail-config.yaml

@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log

server/setup-sarthi.sh

@@ -9,7 +9,7 @@ NC='\033[0m' # No Color
 # Emojis
 CHECK_MARK='✔️'
 CROSS_MARK='❌'
-WARNING='⚠️'
+POINT='👉'
 DOCKER='🐳'
 
 # Check if script is run as root
@@ -18,31 +18,62 @@ if [ "$EUID" -ne 0 ]; then
     exit 1
 fi
 
-# Update package index
-echo -e "${YELLOW}${WARNING} Updating package index...${NC}"
-apt update
+# # Update package index
+# echo -e "${YELLOW}${POINT} Updating package index...${NC}"
+# apt update
 
-# Install prerequisites
-echo -e "${YELLOW}${WARNING} Installing prerequisites...${NC}"
-apt install -y apt-transport-https ca-certificates curl software-properties-common
+# # Install prerequisites
+# echo -e "${YELLOW}${POINT} Installing prerequisites...${NC}"
+# apt install -y apt-transport-https ca-certificates curl software-properties-common
 
-# Install Docker
-echo -e "${YELLOW}${WARNING} Installing Docker...${NC}"
-curl -fsSL https://get.docker.com -o get-docker.sh
-sh get-docker.sh
-rm get-docker.sh
+# # Install Docker
+# echo -e "${YELLOW}${POINT} Installing Docker...${NC}"
+# curl -fsSL https://get.docker.com -o get-docker.sh
+# sh get-docker.sh
+# rm get-docker.sh
 
-# Add user to docker group
-echo -e "${YELLOW}${WARNING} Adding user to the docker group...${NC}"
-usermod -aG docker $USER
-newgrp docker
+# # Add user to docker group
+# echo -e "${YELLOW}${POINT} Adding user to the docker group...${NC}"
+# usermod -aG docker $USER
+# newgrp docker
 
+# # Install Docker Compose
+# echo -e "${YELLOW}${POINT} Installing Docker Compose...${NC}"
+# curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+# chmod +x /usr/local/bin/docker-compose
 
-# Install Docker Compose
-echo -e "${YELLOW}${WARNING} Installing Docker Compose...${NC}"
-curl -sSL https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
-chmod +x /usr/local/bin/docker-compose
+# Start Grafaa + Loki services
+echo -e "${YELLOW}${POINT} Starting Loki + Grafana to export logs ${NC}"
+docker-compose up -d promtail loki grafana
+
+# Install Loki Docker Driver
+echo -e "${YELLOW}${POINT} Installing Loki Docker Driver...${NC}"
+docker plugin install grafana/loki-docker-driver:2.9.1 --alias loki --grant-all-permissions
+
+# Configure Docker Daemon for Loki Logging
+echo -e "${YELLOW}${POINT} Configuring Docker Daemon for Loki Logging...${NC}"
+cat <<EOF > /etc/docker/daemon.json
+{
+    "debug" : true,
+    "log-driver": "loki",
+    "log-opts": {
+        "loki-url": "http://localhost:3100/loki/api/v1/push",
+        "loki-batch-size": "400",
+        "loki-retries": "1",
+        "loki-max-backoff": "500ms",
+        "loki-timeout": "1s",
+        "keep-file": "true"
+    }
+}
+EOF
+
+# Restart Docker Daemon
+echo -e "${YELLOW}${POINT} Restarting Docker Daemon...${NC}"
+systemctl restart docker
+
+echo -e "${YELLOW}${POINT} Recreate All Docker Services to register logger changes${NC}"
+docker-compose up -d --force-recreate
 
 # Display success message
-echo -e "${GREEN}${CHECK_MARK} Docker and Docker Compose installed successfully.${NC}"
+echo -e "${GREEN}${CHECK_MARK} Docker, Docker Compose, and Loki Docker Driver installed and configured successfully.${NC}"
 echo -e "${DOCKER} ${YELLOW}You may need to restart your shell or log out and log back in to apply the changes.${NC}"