Skip to content

Commit

Permalink
Directly get snapshot info (#1)
Browse files Browse the repository at this point in the history
  • Loading branch information
marriva authored Oct 25, 2021
1 parent 2c6d86e commit a9ae5a9
Show file tree
Hide file tree
Showing 10 changed files with 722 additions and 398 deletions.
50 changes: 50 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Docker Image

on:
push:
tags:
- '**'
branches:
- '**'

jobs:

build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: ghcr.io/${{ github.repository_owner }}/elasticsearch-snapshot-exporter
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1

- name: Login to Github Packages
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
20 changes: 13 additions & 7 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,23 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.15
go-version: 1.16

- name: Install promu package
run: go get -v github.com/prometheus/promu

- name: Build
env:
GOOS: linux
GOARCH: amd64
PREFIX: build
run: make
run: |
for GOOS in linux; do
for GOARCH in amd64; do
echo "Building $GOOS-$GOARCH"
export GOOS=$GOOS
export GOARCH=$GOARCH
promu build -v --prefix=build
mv build/es-snapshot-exporter build/es-snapshot-exporter-${GOARCH}-${GOOS}
done
done
promu -v checksum build/
- name: Release
uses: softprops/action-gh-release@v1
Expand Down
27 changes: 13 additions & 14 deletions .promu.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
go:
cgo: false
repository:
path: github.com/flant/elasticsearch-snapshot-exporter
path: github.com/flant/elasticsearch-snapshot-exporter
build:
prefix: build
binaries:
- name: es-snapshot-exporter
ldflags: |
-X github.com/prometheus/common/version.Version={{.Version}}
-X github.com/prometheus/common/version.Revision={{.Revision}}
-X github.com/prometheus/common/version.Branch={{.Branch}}
-X github.com/prometheus/common/version.BuildUser={{user}}@{{host}}
-X github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}}
crossbuild:
platforms:
- linux/amd64
- linux/386
binaries:
- name: es-snapshot-exporter
flags: -a -tags netgo
ldflags: |
-s
-X github.com/prometheus/common/version.Version={{.Version}}
-X github.com/prometheus/common/version.Revision={{.Revision}}
-X github.com/prometheus/common/version.Branch={{.Branch}}
-X github.com/prometheus/common/version.BuildUser={{user}}@{{host}}
-X github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}}
29 changes: 17 additions & 12 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
FROM debian:stable
LABEL maintainer="Vasily Maryutenkov <[email protected]>"
FROM golang:1.16-buster as builder

WORKDIR /src

COPY . .

RUN apt-get update \
&& apt-get dist-upgrade -y \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl
RUN go get -v github.com/prometheus/promu \
&& promu build -v --prefix build


FROM debian:buster-slim
LABEL maintainer="Vasily Maryutenkov <[email protected]>"

ENV VERSION 0.1.0
ENV DOWNLOAD_URL https://github.com/flant/elasticsearch-snapshot-exporter/releases/download/${VERSION}/es-snapshot-exporter-linux-amd64
RUN DEBIAN_FRONTEND=noninteractive; apt-get update \
&& apt-get install -qy --no-install-recommends \
ca-certificates \
tzdata \
curl

RUN curl -fsSL "$DOWNLOAD_URL" -o /es-snapshot-exporter \
&& chmod 755 /es-snapshot-exporter
COPY --from=builder /src/build/es-snapshot-exporter /es-snapshot-exporter

EXPOSE 9141/tcp

ENTRYPOINT [ "/es-snapshot-exporter" ]
CMD [ "--base.dir", "/data" ]
26 changes: 0 additions & 26 deletions Makefile

This file was deleted.

30 changes: 30 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,33 @@
# elasticsearch-snapshot-exporter

[![Go](https://github.com/flant/elasticsearch-snapshot-exporter/actions/workflows/go.yml/badge.svg)](https://github.com/flant/elasticsearch-snapshot-exporter/actions/workflows/go.yml)

```
$ es-snapshot-exporter -h
usage: es-snapshot-exporter [<flags>]
Flags:
-h, --help Show context-sensitive help (also try --help-long and --help-man).
--telemetry.addr=":9141" Listen on host:port.
--telemetry.path="/metrics"
URL path for surfacing collected metrics.
--log.level=info Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]
--log.format=json Set the log format. Valid formats: [json, text]
--schedule="0 14 * * *" Cron job schedule for fetching snapshot data.
--address="http://localhost:9200"
Elasticsearch node to use.
--root.ca=/etc/ssl/certs/elk-root-ca.pem
PEM-encoded certificate authorities
--repository="s3-backup" Elasticsearch snapshot repository name.
--insecure Allow insecure server connections when using SSL.
--threads=2 Number of concurrent http requests to Elasticsearch.
--version Show application version.
```

Expose snapshot size metric

```
# HELP elasticsearch_snapshot_stats_size_in_bytes_total Total size of files that are referenced by the snapshot
# TYPE elasticsearch_snapshot_stats_size_in_bytes_total gauge
elasticsearch_snapshot_stats_size_in_bytes_total{prefix="",repository="",snapshot="",state=""} 0
```
107 changes: 107 additions & 0 deletions client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package main

import (
"crypto/tls"
"encoding/json"
"fmt"
"net/http"

elasticsearch "github.com/elastic/go-elasticsearch/v7"
)

type Client struct {
es *elasticsearch.Client
repository string
}

func NewClient(addresses []string, repository string, insecure bool) (*Client, error) {
cfg := elasticsearch.Config{
Addresses: addresses,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: insecure,
},
},
}

es, err := elasticsearch.NewClient(cfg)
if err != nil {
return nil, err
}

return &Client{es, repository}, nil

}

func (c *Client) GetSnapshot(s []string) ([]map[string]interface{}, error) {
log.Debug("Getting snapshots from repository: ", c.repository)
resp, err := c.es.Snapshot.Get(c.repository, s)
if err != nil {
return nil, fmt.Errorf("error getting response: %s", err)
}
defer resp.Body.Close()

if resp.IsError() {
return nil, fmt.Errorf("request failed: %v", resp.String())
}

var r map[string][]interface{}
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
return nil, err
}

m := make([]map[string]interface{}, 0, len(r))
for _, v := range r["snapshots"] {
m = append(m, v.(map[string]interface{}))
}

return m, nil
}

func (c *Client) GetSnapshotStatus(s []string) ([]map[string]interface{}, error) {
log.Debug("Getting snapshot info for: ", s)
resp, err := c.es.Snapshot.Status(
c.es.Snapshot.Status.WithRepository(c.repository),
c.es.Snapshot.Status.WithSnapshot(s...),
)
if err != nil {
return nil, fmt.Errorf("error getting response: %s", err)
}
defer resp.Body.Close()

if resp.IsError() {
return nil, fmt.Errorf("request failed: %v", resp.String())
}

var r map[string][]interface{}
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
return nil, err
}

m := make([]map[string]interface{}, 0, len(r))
for _, v := range r["snapshots"] {
m = append(m, v.(map[string]interface{}))
}

return m, nil
}

func (c *Client) GetInfo() (map[string]interface{}, error) {
log.Debug("Getting cluster info")
resp, err := c.es.Info()
if err != nil {
return nil, fmt.Errorf("error getting response: %s", err)
}
defer resp.Body.Close()

if resp.IsError() {
return nil, fmt.Errorf("request failed: %v", resp.String())
}

var r map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
return nil, err
}

return r, nil
}
24 changes: 14 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
module github.com/flant/elasticsearch-snapshot-exporter

go 1.15
go 1.16

require (
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/common v0.18.0
github.com/prometheus/procfs v0.6.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/tidwall/gjson v1.6.8
github.com/tidwall/pretty v1.1.0 // indirect
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect
google.golang.org/protobuf v1.25.0 // indirect
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
github.com/bombsimon/logrusr/v2 v2.0.1
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/elastic/go-elasticsearch/v7 v7.10.0
github.com/go-logr/logr v1.2.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.32.1
github.com/prometheus/procfs v0.7.3 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.8.1
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6
)
Loading

0 comments on commit a9ae5a9

Please sign in to comment.