Merge pull request #173 from navidys/image_cache

cache scrapping of images

Makefile

@@ -6,7 +6,12 @@ SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 PRE_COMMIT = $(shell command -v bin/venv/bin/pre-commit ~/.local/bin/pre-commit pre-commit | head -n1)
 PKG_MANAGER ?= $(shell command -v dnf yum|head -n1)
 SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z)
-BUILDFLAGS := -mod=vendor $(BUILDFLAGS)
+BUILDFLAGS := -mod=vendor
+BUILDTAGS ?= \
+	$(shell hack/systemd_tag.sh) \
+	$(shell hack/btrfs_installed_tag.sh) \
+	$(shell hack/btrfs_tag.sh)
+
 VERSION = $(shell cat VERSION  | grep VERSION | cut -d'=' -f2)
 REVISION = $(shell cat VERSION  | grep REVISION | cut -d'=' -f2)
 BRANCH=$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
@@ -58,7 +63,7 @@ binary-remote-arm64: ## Build arm64 prometheus-podman-exporter for remote connec
 $(TARGET): $(SRC)
 	@echo "running go build"
 	@mkdir -p $(BIN)
-	$(GO) build $(BUILDFLAGS) -ldflags="-X '$(PKG_PATH)/cmd.buildVersion=$(VERSION)' -X '$(PKG_PATH)/cmd.buildRevision=$(REVISION)' -X '$(PKG_PATH)/cmd.buildBranch=$(BRANCH)'" -o $(BIN)/$(TARGET)
+	$(GO) build $(BUILDFLAGS) -tags "$(BUILDTAGS)" -ldflags="-X '$(PKG_PATH)/cmd.buildVersion=$(VERSION)' -X '$(PKG_PATH)/cmd.buildRevision=$(REVISION)' -X '$(PKG_PATH)/cmd.buildBranch=$(BRANCH)'" -o $(BIN)/$(TARGET)
 
 .PHONY: install
 install:    ## Install prometheus-podman-exporter binary

exporter/exporter.go

@@ -15,9 +15,6 @@ import (
 
 // Start starts prometheus exporter.
 func Start(cmd *cobra.Command, args []string) error {
-	// setup podman registry
-	pdcs.SetupRegistry()
-
 	// setup exporter
 	promlogConfig := &promlog.Config{Level: &promlog.AllowedLevel{}}
 
@@ -76,6 +73,12 @@ func Start(cmd *cobra.Command, args []string) error {
 			</body>
 			</html>`))
 	})
+
+	// setup podman registry
+	pdcs.SetupRegistry()
+	// start podman event streamer and initiate first update.
+	pdcs.StartEventStreamer(logger)
+
 	level.Info(logger).Log("msg", "Listening on", "address", webListen)
 
 	server := &http.Server{

hack/btrfs_installed_tag.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF
+#include <btrfs/ioctl.h>
+EOF
+if test $? -ne 0 ; then
+	echo exclude_graphdriver_btrfs
+fi

hack/btrfs_tag.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF
+#include <btrfs/version.h>
+EOF
+if test $? -ne 0 ; then
+	echo btrfs_noversion
+fi

hack/systemd_tag.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+${CPP:-${CC:-cc} -E} ${CPPFLAGS} - > /dev/null 2> /dev/null << EOF
+#include <systemd/sd-daemon.h>
+EOF
+if test $? -eq 0 ; then
+	echo systemd
+fi

install.md

@@ -39,14 +39,14 @@ prometheus-podman-exporter is using go v1.17 or above.
 
     ```shell
     $ systemctl start --user podman.socket
-    $ podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v $XDG_RUNTIME_DIR/podman/podman.sock:/run/podman/podman.sock --userns=keep-id --security-opt label=disable quay.io/navidys/prometheus-podman-exporter
+    $ podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v $XDG_RUNTIME_DIR/podman/podman.sock:/run/podman/podman.sock -p 9882:9882 --userns=keep-id --security-opt label=disable quay.io/navidys/prometheus-podman-exporter
     ```
 
 * Using unix socket (root):
 
     ```
     # systemctl start podman.socket
-    # podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v /run/podman/podman.sock:/run/podman/podman.sock -u root --security-opt label=disable quay.io/navidys/prometheus-podman-exporter
+    # podman run -e CONTAINER_HOST=unix:///run/podman/podman.sock -v /run/podman/podman.sock:/run/podman/podman.sock -u root -p 9882:9882 --security-opt label=disable quay.io/navidys/prometheus-podman-exporter
     ```
 
 * Using TCP:

pdcs/events.go

@@ -0,0 +1,55 @@
+package pdcs
+
+import (
+	"context"
+	"log"
+
+	"github.com/containers/podman/v4/cmd/podman/registry"
+	"github.com/containers/podman/v4/libpod/events"
+	"github.com/containers/podman/v4/pkg/domain/entities"
+	klog "github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+)
+
+func StartEventStreamer(logger klog.Logger) {
+	var eventOptions entities.EventsOptions
+
+	level.Debug(logger).Log("msg", "starting podman event streamer")
+	level.Debug(logger).Log("msg", "update images")
+	updateImages()
+
+	eventChannel := make(chan *events.Event, 1)
+	eventOptions.EventChan = eventChannel
+	eventOptions.Stream = true
+	eventOptions.Filter = []string{}
+	errChannel := make(chan error)
+
+	go func() {
+		err := registry.ContainerEngine().Events(context.Background(), eventOptions)
+		if err != nil {
+			errChannel <- err
+		}
+	}()
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-eventChannel:
+				if !ok {
+					level.Error(logger).Log("msg", "podman received event not ok")
+
+					continue
+				}
+
+				if event.Type == events.Image {
+					level.Debug(logger).Log("msg", "update images")
+					updateImages()
+				}
+			case err := <-errChannel:
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		}
+	}()
+}

pdcs/image.go

@@ -2,12 +2,21 @@ package pdcs
 
 import (
 	"fmt"
+	"sync"
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/podman/v4/cmd/podman/registry"
 	"github.com/containers/podman/v4/pkg/domain/entities"
 )
 
+var imageRep ImageReport
+
+type ImageReport struct {
+	images    []Image
+	updateErr error
+	repLock   sync.Mutex
+}
+
 // Image implements image's basic information.
 type Image struct {
 	ID         string
@@ -21,13 +30,34 @@ type Image struct {
 
 // Images returns list of images (Image).
 func Images() ([]Image, error) {
-	images := make([]Image, 0)
+	imageRep.repLock.Lock()
+	defer imageRep.repLock.Unlock()
+
+	if imageRep.updateErr != nil {
+		return nil, imageRep.updateErr
+	}
+
+	images := imageRep.images
+
+	return images, nil
+}
 
+func updateImages() {
+	images := make([]Image, 0)
 	reports, err := registry.ImageEngine().List(registry.GetContext(), entities.ImageListOptions{All: true})
+
+	imageRep.repLock.Lock()
+	defer imageRep.repLock.Unlock()
+
 	if err != nil {
-		return images, err
+		imageRep.updateErr = err
+		imageRep.images = nil
+
+		return
 	}
 
+	imageRep.updateErr = nil
+
 	for _, rep := range reports {
 		if len(rep.RepoTags) > 0 {
 			for i := 0; i < len(rep.RepoTags); i++ {
@@ -56,7 +86,7 @@ func Images() ([]Image, error) {
 		}
 	}
 
-	return images, nil
+	imageRep.images = images
 }
 
 func repoTagDecompose(repoTags string) (string, string) {

prometheus-podman-exporter.spec.rpkg

@@ -50,9 +50,6 @@ BuildRequires: shadow-utils-subid-devel
 
 %build
 %set_build_flags
-%if 0%{?rhel} >= 9
-export BUILDFLAGS="-tags \"exclude_graphdriver_btrfs\""
-%endif
 make binary
 
 %install