container: Prebuild oxidized gem

- Prebuilding the oxidized gem saves some more space
- Added a gitserver to podman-compose to test rugged is installed with
ssh-support
- podman-compose now builds its own oxidized container, as we want to
test the code in the current Git repository.
- update phusion/baseimage to noble

Dockerfile

@@ -1,5 +1,7 @@
-# Stage 1: Build x25519 and any necessary dependencies
-FROM docker.io/phusion/baseimage:noble-1.0.0 AS x25519-builder
+###################
+# Stage 1: Prebuild to save space in the final image.
+
+FROM docker.io/phusion/baseimage:noble-1.0.0 AS prebuilder
 
 # install necessary packages for building gems
 RUN apt-get update && apt-get install -y \
@@ -12,9 +14,23 @@ RUN apt-get update && apt-get install -y \
 RUN mkdir -p /usr/local/bundle
 ENV GEM_HOME=/usr/local/bundle
 
+###################
 # Install the x25519 gem
 RUN gem install x25519 --no-document
 
+###################
+# build oxidized
+COPY . /tmp/oxidized/
+WORKDIR /tmp/oxidized
+
+# docker automated build gets shallow copy, but non-shallow copy cannot be unshallowed
+RUN git fetch --unshallow || true
+
+# Ensure rugged is built with ssh support
+RUN rake build
+
+
+###################
 # Stage2: build an oxidized container from phusion/baseimage-docker and install x25519 from stage1
 FROM docker.io/phusion/baseimage:noble-1.0.0
 
@@ -71,7 +87,7 @@ RUN apt-get -yq update \
     && rm -rf /var/lib/apt/lists/*
 
 # copy the compiled gem from the builder stage
-COPY --from=x25519-builder /usr/local/bundle /usr/local/bundle
+COPY --from=prebuilder /usr/local/bundle /usr/local/bundle
 
 # Set environment variables for bundler
 ENV GEM_HOME="/usr/local/bundle"
@@ -84,17 +100,13 @@ RUN gem install --no-document \
     # dependencies for specific inputs
     net-tftp
 
-# build and install oxidized
-COPY . /tmp/oxidized/
-WORKDIR /tmp/oxidized
-
-# docker automated build gets shallow copy, but non-shallow copy cannot be unshallowed
-RUN git fetch --unshallow || true
-
-# Ensure rugged is built with ssh support
-RUN CMAKE_FLAGS='-DUSE_SSH=ON' rake install
+# install oxidized from prebuilder
+# The Dockerfile ist version-independent, so use oxidized-*.gem to cach the gem
+RUN mkdir -p /tmp/oxidized
+COPY --from=prebuilder /tmp/oxidized/pkg/oxidized-*.gem /tmp/oxidized/
+RUN gem install /tmp/oxidized/oxidized-*.gem
 
-# web interface
+# install oxidized-web
 RUN gem install oxidized-web --no-document
 
 # clean up

examples/podman-compose/Makefile

@@ -6,12 +6,18 @@ help:
 
 rights:
 	podman unshare chown -R 30000:30000 oxidized-config oxidized-ssh
+	podman unshare chown -R 30001 gitserver/repo.git
 
 clean-rights:
 	podman unshare chown -R 0:0 *
 
-start: rights model-image
-	podman-compose -p oxidized up
+start: gitserver-createrepo rights images
+	if [ -f oxidized-config/config ]; then \
+	  podman-compose -p oxidized up ; \
+	else { \
+	  echo "\n########\noxidized-config/config does not exist"; \
+	  echo "create one or copy an example in the folder"; \
+	} fi
 
 run: start
 
@@ -23,39 +29,76 @@ start-local:
 	if [ -f oxidized-config/config.local ]; then \
 	  cp oxidized-config/config.local oxidized-config/config; \
 	else \
-	  echo "oxidized-config/config.local does not exist"; \
+	  echo "\n########\noxidized-config/config.local does not exist"; \
 	fi
 	$(MAKE) start
 
 stop-local: stop
 	if [ -f oxidized-config/config.local ]; then \
 	  git checkout -- oxidized-config/config; \
 	else \
-	  echo "oxidized-config/config.local does not exist"; \
+	  echo "\n########\noxidized-config/config.local does not exist"; \
 	fi
 
+# creates a container image for the model simulation
 model-image:
-	podman image exists localhost/local/model || \
+	podman image exists local/model || \
 	  podman build -t local/model -f model-simulation/Dockerfile-model .
 
 model-clean:
 	podman rmi local/model
 
-clean: stop-local model-clean
+# creates a container image for gitserver
+gitserver-image:
+	podman image exists local/gitserver || \
+	  podman build -t local/gitserver gitserver/
+
+# create the repo repo.git inside the gitserver mapped volume
+gitserver-createrepo: clean-rights
+	if [ ! -d gitserver/repo.git ]; then \
+	  git init --bare gitserver/repo.git; \
+	fi
+
+gitserver-clean:
+	podman rmi local/gitserver
+	rm -rf gitserver/repo.git
+
+gitserver-getkey:
+	podman exec --user oxidized -t oxidized_oxidized_1 sh -c "ssh-keyscan gitserver > /home/oxidized/.ssh/known_hosts"
+
+# build all helper containter images
+images: model-image gitserver-image oxidized-image
+
+# build the oxidized image from the curent repository
+oxidized-image:
+	podman image exists local/oxidized || \
+	  podman build -t local/oxidized ../../
+
+# removes the oxidized image
+oxidized-image-clean:
+	podman rmi local/oxidized
+
+# run evey clean line, even if the previous fails
+clean:
+	-$(MAKE) stop-local
+	-$(MAKE) model-clean
+	-$(MAKE) gitserver-clean
+	-$(MAKE) oxidized-image-clean
 
 define HELP
 make help           - This help
-make rights         - Change the rights of mapped folders for user oxidized
-                      in the container
+make rights         - Change the rights of mapped folders for the users inside
+                      the container
 make clean-rights   - Revert the rights of mapped folders to the local user
-make start          - Start the containter
+make start          - Start the pod with all containers (alias - make run)
                       You can interrupt with Ctrl-C, but make sure you run
-                      make stop to realy stop the container
-make run            - Same as make start
-make stop           - Stop the containter
-make start-local    - Starts the container with the local configuration config.local
-make stop-local     - Stops the container and restores oxidized-config/config from git
-make model-image    - Creates a local OCI-Image to run simulated devices
-make model-clean    - Removes the local OCI-Image to run simulated devices
-make clean          - make stop-local + model-clean
+                      'make stop' to realy stop the container
+make stop           - Stop the pod
+make start-local    - Starts the pod with the local configuration
+                      oxidized-config/config.local
+make stop-local     - Stops the pod and restores
+                      oxidized-config/config from git
+make gitserver-getkey - stores the public key of the gitserver into
+                      oxidized-ssh/known_hosts (the pod must be running)
+make clean          - reverts everything to its original state
 endef

examples/podman-compose/README.md

@@ -1,35 +1,47 @@
-# Running oxidized with podman-compose
-This is an example of Oxidized running within an OCI container, provided by
-podman and podman-compose.
+# Running Oxidized with podman-compose
+This example demonstrates running Oxidized within an OCI container using
+podman-compose. It’s actively used in Oxidized development to validate the
+container’s functionality and to simulate potential issues.
 
-In order to have the example work out of the box, a network device is simulated.
-The model asternos has been chosen because there were not too many commands to
-implement.
+While this example uses podman and podman-compose, it should also be compatible
+with docker, as podman supports docker’s CLI.
 
-To run the example, just run `make start`. You should be sure to have installed the
-[dependencies](#dependencies) before.
+To make this example work seamlessly, a simulated network device is included.
+The asternos model is used here for simplicity, as it requires minimal commands
+to implement. The simulated output doesn’t replicate real device responses but
+provides changing lines over time to test Oxidized’s functionality.
 
-To exit, press `CTRL-C` or run `make stop` in a separate shell. If you exit
-with `CTRL-C`, make sure to run `make stop` after it, in order to clean up the
-running environment.
+
+The example also provides a Git server to test the interaction with it.
+
+# Run the example
+> :warning: the example builds local containers and will require at least 2 GB
+> of disk space along with some CPU and time during the first run.
+
+To start the example, simply run `make start`. Ensure you have installed the
+necessary [dependencies](#dependencies) before.
+
+To stop, press `CTRL-C` or run `make stop` in a separate shell. If you exit
+with `CTRL-C`, make sure to run `make stop` afterward to properly clean up the
+environment.
 
 ## Running Environment
-This example of oxidized with podman-compose has been run on Debian
-Bookworm (Version 12), but should work with few adaptations on any Linux
+This example of oxidized with podman-compose is running on Debian
+Bookworm (Version 12). It should work with few adaptations on any Linux
 box running podman, and maybe also with docker.
 
 ## Dependencies
-You need to install some packages on your debian system:
+To get started, install the required packages on your Debian system:
 ```shell
 sudo apt install podman containers-storage podman-compose make
 ```
 
-You also want to make sure that podman uses the overlay driver for storing its images.
-If not, it will save every layer of the container to disk (and not only the delta),
-so it will fill your disk very fast.
+Ensure Podman is using the overlay driver for image storage.
+Without this driver, Podman may save every container layer separately rather
+than only the changes, which can quickly consume disk space.
 
-This happens if you run podman without having installed the package `container-storage`
-before.
+This issue can occur if podman was run before installing the
+`container-storage`  package.
 
 ```shell
 podman info | grep graphDriverName
@@ -43,16 +55,40 @@ You should get this reply
 If not, the quick way I found to solve it is to delete `~/.local/share/containers/`.
 Beware - this will delete **all** your containers!
 
-## I want to adapt this to my needs
-Feel free and have fun. You probably want to edit docker-compose.yml in order to remove the
-simulated model.
+## Adapting to your needs
+Feel free to customize this setup as you wish! You may want to edit
+`docker-compose.yml` to remove any containers simulating specific components.
 
-## Use your own oxidized configuration within the git repository
-When developing oxidized and testing the container, you may want to use your
-own configuration. This can be done by saving it under `oxidized-config/config.local`
+## Use your own oxidized configuration in the git repository
+When developing oxidized or testing the container, you may want to use a custom
+configuration. This can be done by saving it under `oxidized-config/config.local`
 
 `make start-local` will recognize the local configuration and copy it to
 `oxidized-config/config` before starting the container.
 
-You shoud stop the container with `make stop-local` in order to restore the original
-configuration from git.
+You should stop the container with `make stop-local` in order to restore the
+original configuration from the git repository.
+
+In the folder `oxidized-config/, you will also find some example configs,
+for example `config_csv-gitserver`. To use them, just copy the file to `config`.
+
+## Git server public keys
+To enable Oxidized to access the Git server, you'll need to retrieve the
+servers' public SSH keys and store them under `oxidized-ssh/known_hosts`.
+Without this, you will encounter the following error:
+
+```
+ERROR -- : Hook push_to_remote (#<GithubRepo:0x00007f4cff47d918>) failed (#<Rugged::SshError: invalid or unknown remote ssh hostkey>) for event :post_store
+```
+
+While the container environment is running (`make start`), open a separate shell
+and run:
+```
+make gitserver-getkey
+```
+
+You do not need to restart the container environment; Oxidized will
+automatically use the key the next time it pushes to the remote Git repository.
+
+
+

examples/podman-compose/docker-compose.yml

@@ -1,8 +1,10 @@
 services:
   oxidized:
     # Choose the image that you want to test
-    # image: docker.io/oxidized/oxidized:0.29.1
-    image: docker.io/oxidized/oxidized:latest
+    # image: docker.io/oxidized/oxidized:0.30.1
+    # image: docker.io/oxidized/oxidized:latest
+    # local/oxidized is build by "make oxidized-image" and "make run"
+    image: local/oxidized
     ports:
       - 127.0.0.1:8042:8888/tcp
     environment:
@@ -13,9 +15,16 @@ services:
     volumes:
        - ./oxidized-config:/home/oxidized/.config/oxidized
        - ./oxidized-ssh:/home/oxidized/.ssh
+
   # This is a simulated network device for the example to work out of the box
   asternos-device:
     image: localhost/local/model
     volumes:
       - ./model-simulation/asternos.sh:/home/oxidized/.profile
       - ./model-simulation/asternos.sh:/home/admin/.profile
+
+  # This is a gitserver to push our configs
+  gitserver:
+    image: localhost/local/gitserver
+    volumes:
+      - ./gitserver/repo.git:/home/git/repo.git

examples/podman-compose/gitserver/.gitignore

@@ -0,0 +1 @@
+repo.git

examples/podman-compose/gitserver/Dockerfile

@@ -0,0 +1,14 @@
+FROM docker.io/phusion/baseimage:noble-1.0.0
+
+# Use baseimage-docker's init system.
+CMD ["/sbin/my_init"]
+
+# enable ssh
+RUN rm -f /etc/service/sshd/down
+RUN /etc/my_init.d/00_regen_ssh_host_keys.sh
+
+# Add user for the gitserver. The password is "git"
+RUN useradd -m git -p '$6$32WDb0LTFyQkLffy$u15COVx7CQ4tgp4JT4DO4LJ96q/jwFSpuZC3WrllNQDNa6nW1LhJKW9rLV57ak3rj9Ln./aRA85jzeof1B0Gi1' -s /bin/bash -u 30001
+
+# And install git
+RUN install_clean git

examples/podman-compose/model-simulation/Dockerfile-model

@@ -1,4 +1,4 @@
-FROM docker.io/phusion/baseimage:jammy-1.0.2
+FROM docker.io/phusion/baseimage:noble-1.0.0
 
 # Use baseimage-docker's init system.
 CMD ["/sbin/my_init"]

examples/podman-compose/model-simulation/asternos.sh

@@ -16,6 +16,8 @@ EOF
 function show() {
   if [ "$*" == "version" ]; then
     echo "Version 1.2.3"
+    # Make the output change over time
+    date
   elif [ "$*" == "runningconfiguration all" ]; then
           cat << EOF
 ! begin of the configuration

examples/podman-compose/oxidized-config/.gitignore

@@ -7,3 +7,4 @@ pid
 configs/
 crash
 logs/
+oxidized.git/