Added monitoring containers

alertmanager/config.yml

@@ -0,0 +1,11 @@
+route:
+    receiver: 'slack'
+
+receivers:
+    - name: 'slack'
+      slack_configs:
+          - send_resolved: true
+            text: "{{ .CommonAnnotations.description }}"
+            username: 'Prometheus'
+            channel: '#prometheus'
+            api_url: 'https://hooks.slack.com/services/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'

docker-compose.yml

@@ -1,20 +1,78 @@
-version: "3.9"
+version: '2.1'
+
+networks:
+  monitor-net:
+    driver: bridge
+
+volumes:
+    prometheus_data: {}
+    grafana_data: {}
+    pgdata:
+       external: true
+       name: balsam_pgdata
 
 services:
+
+
+  balsam-web:
+    image: anl/balsam-web
+    container_name: balsam-web
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/balsam
+    networks:
+      - monitor-net
+    ports:
+      - 5000:5000
+
+  balsam-ui:
+    image: anl/balsam-ui
+    container_name: balsam-ui
+    restart: always
+    depends_on:
+      - balsam-web
+    networks:
+      - monitor-net
+    ports:
+      - 8007:8080
+
+  pgadmin:
+    image: dpage/pgadmin4
+    container_name: pgadmin
+    networks:
+      - monitor-net
+    environment:
+      PGADMIN_DEFAULT_EMAIL: [email protected]
+      PGADMIN_DEFAULT_PASSWORD: y9lhFN1XbnVMNRyI
+    ports:
+      - 8008:80
+
+  portainer:
+    image: portainer/portainer-ce:latest
+    container_name: portainer
+    networks:
+      - monitor-net
+    ports:
+      - 9000:9000
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
   gunicorn:
     container_name: gunicorn
     build: .
-    image: masalim2/balsam
+    image: anl/balsam
     restart: always
+    networks:
+      - monitor-net
     ports:
       - ${SERVER_PORT}:${SERVER_PORT}
     depends_on:
       postgres:
         condition: service_healthy
       redis:
         condition: service_started
-    # Vars in env_file are exported to the containers
-    # Vars in ".env" specifically are also usable in the compose file as ${VAR}
     env_file: ".env"
     environment:
       SERVER_PORT: ${SERVER_PORT}
@@ -38,6 +96,8 @@ services:
     volumes:
       - "pgdata:/var/lib/postgresql/data"
     command: "-c log_min_duration_statement=0"
+    networks:
+      - monitor-net
     logging:
       options:
         max-size: "50m"
@@ -50,11 +110,154 @@ services:
       retries: 6
 
   redis:
+    networks:
+      - monitor-net
     container_name: redis
     image: redis
     restart: always
     ports:
       - 6379:6379
 
-volumes:
-  pgdata:
+  statsd:
+    container_name: statsd
+    image: prom/statsd-exporter
+    command: 
+      - '--statsd.mapping-config=/tmp/statsd_mapping.conf'
+    networks:
+      - monitor-net
+    ports:
+      - "9102:9102"
+      - "9125:9125/udp"
+    volumes:
+      - "./statsd.conf:/tmp/statsd_mapping.conf"
+
+  pushgateway:
+    image: prom/pushgateway:v1.2.0
+    container_name: pushgateway
+    restart: unless-stopped
+    expose:
+      - 9091
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  cadvisor:
+    image: gcr.io/google-containers/cadvisor:v0.34.0
+    container_name: cadvisor
+    networks:
+      - monitor-net
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+    ports:
+      - 8080:8080
+    labels:
+      org.label-schema.group: "monitoring"
+    restart: always
+
+  nginxexporter:
+    image: nginx/nginx-prometheus-exporter:0.10.0 
+    container_name: nginxexporter
+    command: -nginx.scrape-uri=https://balsam-dev.alcf.anl.gov/basic_status
+    networks:
+      - monitor-net
+    ports:
+      - 9113:9113
+    labels:
+      org.label-schema.group: "monitoring"
+
+  nginx-exporter:
+    image: quay.io/martinhelmich/prometheus-nginxlog-exporter
+    container_name: nginx-exporter
+    networks:
+      - monitor-net
+    volumes:
+      - /var/log/nginx:/mnt/nginxlogs
+      - /home/dgovoni/balsam.hcl:/etc/prometheus-nginxlog-exporter.hcl
+    command: -config-file /etc/prometheus-nginxlog-exporter.hcl
+    ports:
+      - 4040:4040
+    labels:
+      org.label-schema.group: "monitoring"
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus:/etc/prometheus
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--storage.tsdb.retention.time=200h'
+      - '--web.enable-lifecycle'
+    restart: unless-stopped
+    networks:
+      - monitor-net
+    ports:
+      - 9090:9090
+    labels:
+      org.label-schema.group: "monitoring"
+
+  alertmanager:
+    image: prom/alertmanager:v0.20.0
+    container_name: alertmanager
+    volumes:
+      - ./alertmanager:/etc/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/config.yml'
+      - '--storage.path=/alertmanager'
+    restart: unless-stopped
+    networks:
+      - monitor-net
+    ports:
+      - 9093:9093
+    labels:
+      org.label-schema.group: "monitoring"
+
+  nodeexporter:
+    image: prom/node-exporter:v0.18.1
+    container_name: nodeexporter
+    networks:
+      - monitor-net
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+    restart: unless-stopped
+    expose:
+      - 9100
+    labels:
+      org.label-schema.group: "monitoring"
+
+  grafana:
+    image: grafana/grafana:6.7.2
+    container_name: grafana
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    networks:
+      - monitor-net
+    environment:
+      - GF_SECURITY_ADMIN_USER=${ADMIN_USER}
+      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD}
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_SMTP_ENABLED=true
+      - GF_SMTP_HOST=smtp.anl.gov:587
+      - [email protected]
+      - GF_SMTP_PASSWORD=Shelby!1965!GT350!
+    ports:
+      - 3000:3000
+    labels:
+      org.label-schema.group: "monitoring"
+

prometheus/prometheus.yml

@@ -0,0 +1,69 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'docker-host-alpha'
+
+# Load and evaluate rules in this file every 'evaluation_interval' seconds.
+rule_files:
+  - "alert.rules"
+
+# A scrape configuration containing exactly one endpoint to scrape.
+scrape_configs:
+  - job_name: 'nodeexporter'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['nodeexporter:9100']
+
+  - job_name: 'nginx'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['nginx-exporter:4040']
+
+  - job_name: 'cadvisor'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: 'prometheus'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'pushgateway'
+    scrape_interval: 10s
+    honor_labels: true
+    static_configs:
+      - targets: ['pushgateway:9091']
+
+  - job_name: 'balsam_gunicorn'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['statsd:9102']
+
+  - job_name: 'nginx2'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['nginxexporter:9113']
+
+
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets: 
+      - 'alertmanager:9093'
+
+#  - job_name: 'nginx'
+#    scrape_interval: 10s
+#    static_configs:
+#      - targets: ['nginxexporter:9113']
+
+#  - job_name: 'aspnetcore'
+#    scrape_interval: 10s
+#    static_configs:
+#      - targets: ['eventlog-proxy:5000', 'eventlog:5000']