From 31368b9df3e4dd3c84505ef6faf79a00a369f97b Mon Sep 17 00:00:00 2001 From: Navid Yaghoobi Date: Sun, 28 Jan 2024 20:26:18 +1100 Subject: [PATCH] cache scrapping of images Signed-off-by: Navid Yaghoobi --- Makefile | 9 ++++- exporter/exporter.go | 9 +++-- hack/btrfs_installed_tag.sh | 7 ++++ hack/btrfs_tag.sh | 7 ++++ hack/systemd_tag.sh | 7 ++++ install.md | 4 +- pdcs/events.go | 55 ++++++++++++++++++++++++++++ pdcs/image.go | 36 ++++++++++++++++-- prometheus-podman-exporter.spec.rpkg | 3 -- 9 files changed, 124 insertions(+), 13 deletions(-) create mode 100755 hack/btrfs_installed_tag.sh create mode 100755 hack/btrfs_tag.sh create mode 100755 hack/systemd_tag.sh create mode 100644 pdcs/events.go diff --git a/Makefile b/Makefile index afe003fa..b34578e6 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,12 @@ SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*") PRE_COMMIT = $(shell command -v bin/venv/bin/pre-commit ~/.local/bin/pre-commit pre-commit | head -n1) PKG_MANAGER ?= $(shell command -v dnf yum|head -n1) SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z) -BUILDFLAGS := -mod=vendor $(BUILDFLAGS) +BUILDFLAGS := -mod=vendor +BUILDTAGS ?= \ + $(shell hack/systemd_tag.sh) \ + $(shell hack/btrfs_installed_tag.sh) \ + $(shell hack/btrfs_tag.sh) + VERSION = $(shell cat VERSION | grep VERSION | cut -d'=' -f2) REVISION = $(shell cat VERSION | grep REVISION | cut -d'=' -f2) BRANCH=$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) @@ -58,7 +63,7 @@ binary-remote-arm64: ## Build arm64 prometheus-podman-exporter for remote connec $(TARGET): $(SRC) @echo "running go build" @mkdir -p $(BIN) - $(GO) build $(BUILDFLAGS) -ldflags="-X '$(PKG_PATH)/cmd.buildVersion=$(VERSION)' -X '$(PKG_PATH)/cmd.buildRevision=$(REVISION)' -X '$(PKG_PATH)/cmd.buildBranch=$(BRANCH)'" -o $(BIN)/$(TARGET) + $(GO) build $(BUILDFLAGS) -tags "$(BUILDTAGS)" -ldflags="-X '$(PKG_PATH)/cmd.buildVersion=$(VERSION)' -X '$(PKG_PATH)/cmd.buildRevision=$(REVISION)' -X '$(PKG_PATH)/cmd.buildBranch=$(BRANCH)'" -o $(BIN)/$(TARGET) .PHONY: install install: ## Install prometheus-podman-exporter binary diff --git a/exporter/exporter.go b/exporter/exporter.go index f607fa93..e0934aed 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -15,9 +15,6 @@ import ( // Start starts prometheus exporter. func Start(cmd *cobra.Command, args []string) error { - // setup podman registry - pdcs.SetupRegistry() - // setup exporter promlogConfig := &promlog.Config{Level: &promlog.AllowedLevel{}} @@ -76,6 +73,12 @@ func Start(cmd *cobra.Command, args []string) error { `)) }) + + // setup podman registry + pdcs.SetupRegistry() + // start podman event streamer and initiate first update. + pdcs.StartEventStreamer(logger) + level.Info(logger).Log("msg", "Listening on", "address", webListen) server := &http.Server{ diff --git a/hack/btrfs_installed_tag.sh b/hack/btrfs_installed_tag.sh new file mode 100755 index 00000000..f2f2b33c --- /dev/null +++ b/hack/btrfs_installed_tag.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF +#include +EOF +if test $? -ne 0 ; then + echo exclude_graphdriver_btrfs +fi diff --git a/hack/btrfs_tag.sh b/hack/btrfs_tag.sh new file mode 100755 index 00000000..ea753d4d --- /dev/null +++ b/hack/btrfs_tag.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF +#include +EOF +if test $? -ne 0 ; then + echo btrfs_noversion +fi diff --git a/hack/systemd_tag.sh b/hack/systemd_tag.sh new file mode 100755 index 00000000..5af32288 --- /dev/null +++ b/hack/systemd_tag.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF +#include +EOF +if test $? -eq 0 ; then + echo systemd +fi diff --git a/install.md b/install.md index 29620240..02bf34f1 100644 --- a/install.md +++ b/install.md @@ -39,14 +39,14 @@ prometheus-podman-exporter is using go v1.17 or above. ```shell $ systemctl start --user podman.socket - $ podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v $XDG_RUNTIME_DIR/podman/podman.sock:/run/podman/podman.sock --userns=keep-id --security-opt label=disable quay.io/navidys/prometheus-podman-exporter + $ podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v $XDG_RUNTIME_DIR/podman/podman.sock:/run/podman/podman.sock -p 9882:9882 --userns=keep-id --security-opt label=disable quay.io/navidys/prometheus-podman-exporter ``` * Using unix socket (root): ``` # systemctl start podman.socket - # podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v /run/podman/podman.sock:/run/podman/podman.sock -u root --security-opt label=disable quay.io/navidys/prometheus-podman-exporter + # podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v /run/podman/podman.sock:/run/podman/podman.sock -u root -p 9882:9882 --security-opt label=disable quay.io/navidys/prometheus-podman-exporter ``` * Using TCP: diff --git a/pdcs/events.go b/pdcs/events.go new file mode 100644 index 00000000..5e2794d6 --- /dev/null +++ b/pdcs/events.go @@ -0,0 +1,55 @@ +package pdcs + +import ( + "context" + "log" + + "github.com/containers/podman/v4/cmd/podman/registry" + "github.com/containers/podman/v4/libpod/events" + "github.com/containers/podman/v4/pkg/domain/entities" + klog "github.com/go-kit/log" + "github.com/go-kit/log/level" +) + +func StartEventStreamer(logger klog.Logger) { + var eventOptions entities.EventsOptions + + level.Debug(logger).Log("msg", "starting podman event streamer") + level.Debug(logger).Log("msg", "update images") + updateImages() + + eventChannel := make(chan *events.Event, 1) + eventOptions.EventChan = eventChannel + eventOptions.Stream = true + eventOptions.Filter = []string{} + errChannel := make(chan error) + + go func() { + err := registry.ContainerEngine().Events(context.Background(), eventOptions) + if err != nil { + errChannel <- err + } + }() + + go func() { + for { + select { + case event, ok := <-eventChannel: + if !ok { + level.Error(logger).Log("msg", "podman received event not ok") + + continue + } + + if event.Type == events.Image { + level.Debug(logger).Log("msg", "update images") + updateImages() + } + case err := <-errChannel: + if err != nil { + log.Fatal(err) + } + } + } + }() +} diff --git a/pdcs/image.go b/pdcs/image.go index 89329b73..1557e8cc 100644 --- a/pdcs/image.go +++ b/pdcs/image.go @@ -2,12 +2,21 @@ package pdcs import ( "fmt" + "sync" "github.com/containers/image/v5/docker/reference" "github.com/containers/podman/v4/cmd/podman/registry" "github.com/containers/podman/v4/pkg/domain/entities" ) +var imageRep ImageReport + +type ImageReport struct { + images []Image + updateErr error + repLock sync.Mutex +} + // Image implements image's basic information. type Image struct { ID string @@ -21,13 +30,34 @@ type Image struct { // Images returns list of images (Image). func Images() ([]Image, error) { - images := make([]Image, 0) + imageRep.repLock.Lock() + defer imageRep.repLock.Unlock() + + if imageRep.updateErr != nil { + return nil, imageRep.updateErr + } + + images := imageRep.images + + return images, nil +} +func updateImages() { + images := make([]Image, 0) reports, err := registry.ImageEngine().List(registry.GetContext(), entities.ImageListOptions{All: true}) + + imageRep.repLock.Lock() + defer imageRep.repLock.Unlock() + if err != nil { - return images, err + imageRep.updateErr = err + imageRep.images = nil + + return } + imageRep.updateErr = nil + for _, rep := range reports { if len(rep.RepoTags) > 0 { for i := 0; i < len(rep.RepoTags); i++ { @@ -56,7 +86,7 @@ func Images() ([]Image, error) { } } - return images, nil + imageRep.images = images } func repoTagDecompose(repoTags string) (string, string) { diff --git a/prometheus-podman-exporter.spec.rpkg b/prometheus-podman-exporter.spec.rpkg index a6f51f74..bb08887c 100644 --- a/prometheus-podman-exporter.spec.rpkg +++ b/prometheus-podman-exporter.spec.rpkg @@ -50,9 +50,6 @@ BuildRequires: shadow-utils-subid-devel %build %set_build_flags -%if 0%{?rhel} >= 9 -export BUILDFLAGS="-tags \"exclude_graphdriver_btrfs\"" -%endif make binary %install