From ed3f906c7afaae92e6b52860ffbf2b8c41c8c873 Mon Sep 17 00:00:00 2001 From: mbshields Date: Fri, 16 Jun 2023 15:01:08 -0700 Subject: [PATCH 1/4] docs: update docs with v1.0.0 changes Signed-off-by: mbshields --- docs/reference/stacker_cli.md | 48 +++++++++++++++------ docs/reference/stacker_file.md | 2 +- docs/user_guide/annotation_support.md | 2 +- docs/user_guide/convert_dockerfile.md | 32 ++++++++++++++ docs/user_guide/publish_images.md | 5 +-- docs/user_guide/scratch_image.md | 2 +- docs/user_guide/template_substitution.md | 4 ++ docs/whats-new.md | 53 ++++++++++++++---------- mkdocs.yml | 1 + 9 files changed, 107 insertions(+), 42 deletions(-) create mode 100644 docs/user_guide/convert_dockerfile.md diff --git a/docs/reference/stacker_cli.md b/docs/reference/stacker_cli.md index f186550..a153fe4 100644 --- a/docs/reference/stacker_cli.md +++ b/docs/reference/stacker_cli.md @@ -1,18 +1,21 @@ # Stacker Command-Line Interface Reference +This document lists the command line interface (CLI) commands for stacker version 1.0.0. + - - - - - - - - + + + + + + + + + - - + +
stackerStacker builds OCI images
stacker buildBuild a new OCI image from a stacker yaml file
stacker recursive-buildFind stacker yaml files under a directory and builds all OCI layers they define
stacker publishPublish OCI images previously built from one or more stacker yaml files
stacker chrootRun a command in a chroot (same as stacker exec)
stacker cleanClean up after a stacker build
stacker inspectPrint the json representation of an OCI image
stacker grabGrab a file from the layer's filesystem
stacker unpriv-setupDo the necessary unprivileged setup for stacker build to work without root
stacker buildBuilds a new OCI image from a stacker yaml file
stacker recursive-buildFinds stacker yaml files under a directory and builds all OCI layers they define
stacker convertConverts a Dockerfile into a stacker yaml file
stacker publishPublishes OCI images previously built from one or more stacker yaml files
stacker chrootRuns a command in a chroot (same as stacker exec)
stacker cleanCleans up after a stacker build
stacker inspectPrints the json representation of an OCI image
stacker grabGrabs a file from the layer's filesystem
stacker unpriv-setupDoes the necessary unprivileged setup for stacker build to work without root
stacker gcGarbage collection of unused OCI imports/outputs snapshots
stacker checkCheck that all runtime required items (like kernel features) are present
stacker helpShow a list of commands or help for one command
stacker checkChecks that all runtime required items (like kernel features) are present
stacker helpShows a list of commands or help for one command
@@ -26,11 +29,12 @@ stacker [global options] command [command options] [arguments...] VERSION: - stacker v0.40.1-e54a685 liblxc a330126b45c7c3b6fcf0f9ba6c1eda7bdb4e508a + stacker v1.0.0-rc4-8e267fc liblxc 727adc05220e9b8b594c5bc2124bef5b2e0abe7a COMMANDS: build builds a new OCI image from a stacker yaml file recursive-build finds stacker yaml files under a directory and builds all OCI layers they define + convert converts a Dockerfile into a stacker yaml file (experimental, best-effort) publish publishes OCI images previously built from one or more stacker yaml files chroot, exec run a command in a chroot clean cleans up after a `stacker build` @@ -42,10 +46,11 @@ help, h Shows a list of commands or help for one command GLOBAL OPTIONS: + --work-dir value set the working directory for stacker's cache, OCI output and rootfs output --stacker-dir value set the directory for stacker's cache (default: ".stacker") --oci-dir value set the directory for OCI output (default: "oci") --roots-dir value set the directory for the rootfs output (default: "roots") - --config value stacker config file with defaults (default: "/users/mishield/.config/stacker/conf.yaml") + --config value stacker config file with defaults (default: "/home//.config/stacker/conf.yaml") --debug enable stacker debug mode -q, --quiet silence all logs --log-file value log to a file instead of stderr @@ -102,6 +107,21 @@ --stacker-file-pattern value, -p value regex pattern to use when searching for stackerfile paths (default: "\\/stacker.yaml$") --search-dir value, -d value directory under which to search for stackerfiles to build (default: ".") + + +## stacker convert + + NAME: + stacker convert - converts a Dockerfile into a stacker yaml file (experimental, best-effort) + + USAGE: + stacker convert [command options] [arguments...] + + OPTIONS: + --docker-file value, -i value the input Dockerfile (default: "Dockerfile") + --output-file value, -o value the output stacker file (default: "stacker.yaml") + --substitute-file value, -s value the output file containing detected substitutions (default: "stacker-subs.yaml") + ## stacker publish @@ -115,6 +135,7 @@ OPTIONS: --stacker-file value, -f value the input stackerfile (default: "stacker.yaml") --stacker-file-pattern value, -p value regex pattern to use when searching for stackerfile paths (default: "\\/stacker.yaml$") + --substitute-file value file containing variable substitution in stackerfiles, 'FOO: bar' yaml format --search-dir value, -d value directory under which to search for stackerfiles to publish --url value url where to publish the OCI images --username value username for the registry where the OCI images are published @@ -125,6 +146,7 @@ --show-only show the images to be published without actually publishing them --force force publishing the images present in the OCI layout even if they should be rebuilt --layer-type value set the output layer type (supported values: tar, squashfs); can be supplied multiple times (default: "tar") + --layer value layer to be published; can be specified multiple times @@ -157,7 +179,7 @@ stacker clean [command options] [arguments...] OPTIONS: - --all no-op; this used to do soemthing, and is left in for compatibility + --all no-op; this used to do something, and is left in for compatibility diff --git a/docs/reference/stacker_file.md b/docs/reference/stacker_file.md index 0be9be1..601b4b8 100644 --- a/docs/reference/stacker_file.md +++ b/docs/reference/stacker_file.md @@ -242,7 +242,7 @@ itself and not as part of the index.json. a.b.c.key: abc_val p.q.r.key: pqr_val -While `config` section supports a similar `labels`, it is more pertitent to the +While the `config` section supports a similar `labels`, it is more pertinent to the image runtime. On the other hand, `annotations` is intended to be image-specific metadata aligned with the [annotations in the image spec](https://github.com/opencontainers/image-spec/blob/main/annotations.md). diff --git a/docs/user_guide/annotation_support.md b/docs/user_guide/annotation_support.md index bfc35f7..14c5a70 100644 --- a/docs/user_guide/annotation_support.md +++ b/docs/user_guide/annotation_support.md @@ -8,7 +8,7 @@ itself and not as part of the index.json. a.b.c.key: abc_val p.q.r.key: pqr_val -While `config` section supports a similar `labels`, it is more pertitent to the +While the `config` section supports a similar `labels`, it is more pertinent to the image runtime. On the other hand, `annotations` is intended to be image-specific metadata aligned with the [annotations in the image spec](https://github.com/opencontainers/image-spec/blob/main/annotations.md). diff --git a/docs/user_guide/convert_dockerfile.md b/docs/user_guide/convert_dockerfile.md new file mode 100644 index 0000000..5766933 --- /dev/null +++ b/docs/user_guide/convert_dockerfile.md @@ -0,0 +1,32 @@ +# Convert a Dockerfile to stacker yaml + +When moving a project from a docker build environment to a stacker build environment, you can use the [`stacker convert`](../reference/stacker_cli.md#stacker-convert) command to create a stacker yaml file from the existing Dockerfile. + +!!! note + + The conversion is a best-effort process and may not be successful in all cases. + +During the conversion, variables from the Dockerfile may be exported to a substitution file, which can be declared in the `stacker build` command when building the project. + +When invoking the `stacker convert` command, you have the option to specify the input (Dockerfile) filename, an output (stacker) filename, and a name for a substitution file, if such a file is generated by the conversion. Alternatively, you can use the default names provided: + + +| Command option | Abbreviation | Default name | +| --- | --- | --- | +| --docker-file | -i | "Dockerfile" | +| --output-file | -o | "stacker.yaml" | +| --substitute-file | -s | "stacker-subs.yaml" | + + +To execute a conversion, use the following syntax: + + stacker convert [-i ] [-o ] [-s ] + + + + + + + + + \ No newline at end of file diff --git a/docs/user_guide/publish_images.md b/docs/user_guide/publish_images.md index 6f4273e..32e8fa5 100644 --- a/docs/user_guide/publish_images.md +++ b/docs/user_guide/publish_images.md @@ -1,8 +1,6 @@ # Publishing images -After building the image, you can publish the image to an [OCI Distribution -Spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) -conformant registry. +After building the image, you can publish the image to an [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) conformant registry. ``` NAME: @@ -24,4 +22,5 @@ OPTIONS: --show-only show the images to be published without actually publishing them --force force publishing the images present in the OCI layout even if they should be rebuilt --layer-type value set the output layer type (supported values: tar, squashfs); can be supplied multiple times (default: "tar") + --layer value layer to be published; can be specified multiple times ``` diff --git a/docs/user_guide/scratch_image.md b/docs/user_guide/scratch_image.md index 84bbe2a..d1b22b6 100644 --- a/docs/user_guide/scratch_image.md +++ b/docs/user_guide/scratch_image.md @@ -25,7 +25,7 @@ The best way to accomplish this is as follows: type: tar url: stacker://build/contents.tar -Or e.g. to bootstrap a base layer for CentoOS 7: +Or e.g. to bootstrap a base layer for CentOS 7: build: from: diff --git a/docs/user_guide/template_substitution.md b/docs/user_guide/template_substitution.md index 069875d..ef817d4 100644 --- a/docs/user_guide/template_substitution.md +++ b/docs/user_guide/template_substitution.md @@ -15,3 +15,7 @@ That is, variables of the form `$FOO` or `${FOO}` are supported, and variables with `${FOO:default}` a default value will evaluate to their default if not specified on the command line. It is an error to specify a `${FOO}` style without a default; to make the default an empty string, use `${FOO:}`. + +You can also declare variable substitutions in a separate file which is then included in the build command as in this example: + +`stacker build --substitute-file ` diff --git a/docs/whats-new.md b/docs/whats-new.md index 541b0a0..5aec019 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -1,36 +1,43 @@ # What's New -## [v0.40.1](https://github.com/project-stacker/stacker/releases/tag/v0.40.1) +## [v1.0.0](https://github.com/project-stacker/stacker/releases/tag/v1.0.0-rc4) -* Support for `scratch` +* Convert a Dockerfile for stacker -Prior to v0.40.1, `stacker` did not support empty root filesystems to be used a -base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host -statically built binaries. + A new [`stacker convert`](reference/stacker_cli.md#stacker-convert) command performs a conversion of a Dockerfile into a stacker.yaml file. During the conversion, some variables from the Dockerfile may be exported to a substitution file. -* Support for `import`ing content into container image + > **NOTE:** The conversion is a best-effort process and may not be successful in all cases. + + +* Publish specific layers + + By default, the [`stacker publish`](reference/stacker_cli.md#stacker-publish) command pushes all layers in a stacker.yaml file instead of only the required layers. Using a new command option, `layer `, you can explicitly specify which layers are to be published. This command option can be specified multiple times, selecting each layer to be included. + +* Specify a single working directory + + A new [`stacker`](reference/stacker_cli.md#stacker) command option, `work-dir`, sets the working directory for stacker's cache, OCI output, and rootfs output. The existing command options `stacker-dir`, `oci-dir`, and `roots-dir` can then be omitted or used to override the `work-dir` setting. -Prior to v0.40.1, copying content into a layer permanently involved bind -mounting a shell such as busybox and invoking appropriate commands using the -`run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve -the same. -* For the `build` command, can specify substitute key-value pairs in a file instead of the commandline. +* Report kernel version and fs type -``` -$ stacker build --help -NAME: - stacker build - builds a new OCI image from a stacker yaml file + The [`stacker check`](reference/stacker_cli.md#stacker-check) command now reports this information. + +* Build improvements + + +## [v0.40.1](https://github.com/project-stacker/stacker/releases/tag/v0.40.1) +* Support for `scratch` + + Prior to v0.40.1, `stacker` did not support empty root filesystems to be used a base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host statically built binaries. + +* Support for `import`ing content into container image -USAGE: - stacker build [command options] [arguments...] + Prior to v0.40.1, copying content into a layer permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. -OPTIONS: -... - --substitute-file value file containing variable substitution in stackerfiles, 'FOO: bar' yaml format -... -``` +* Publish with substitutions specified in a file + + Using a new `build` command option, [`substitute-file `](reference/stacker_cli.md#stacker-build), you can now declare variable substitutions in a file instead of the command line. The substitution file uses a 'FOO: bar' key-value yaml format to declare substitutions. * Some `squashfs` improvements -While building squashfs layers, use `squashfuse_ll` if available which is faster. + While building squashfs layers, use `squashfuse_ll` if available which is faster. diff --git a/mkdocs.yml b/mkdocs.yml index af2728e..81474bf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -122,6 +122,7 @@ nav: - Image Chroot: user_guide/image_chroot.md - Grab Image Content: user_guide/grab_image_content.md - Debugging: user_guide/debugging.md + - Converting a Dockerfile: user_guide/convert_dockerfile.md - Reference: - Stacker CLI: reference/stacker_cli.md - Stacker File: reference/stacker_file.md From 8f2b9d43ba72493ce91a9eeb197f345cc9bdb643 Mon Sep 17 00:00:00 2001 From: mbshields Date: Tue, 5 Dec 2023 10:42:58 -0800 Subject: [PATCH 2/4] docs: update What's New for v1.0.0, add SBOM article Signed-off-by: mbshields --- docs/reference/stacker_file.md | 76 ++++++------- docs/user_guide/generate_sbom.md | 177 +++++++++++++++++++++++++++++++ docs/whats-new.md | 57 ++++++---- mkdocs.yml | 1 + 4 files changed, 251 insertions(+), 60 deletions(-) create mode 100644 docs/user_guide/generate_sbom.md diff --git a/docs/reference/stacker_file.md b/docs/reference/stacker_file.md index 601b4b8..7009a87 100644 --- a/docs/reference/stacker_file.md +++ b/docs/reference/stacker_file.md @@ -63,24 +63,17 @@ layer on a previously specified layer in the stacker file. `scratch`: which is an empty rootfs and can be used to host statically built binaries. -### `import` +### `imports` -The `import` directive describes what files should be made available in -`/stacker` during the `run` phase. There are three forms of importing supported -today: +The `imports` directive describes what files should be made available in `/stacker/imports` during the `run` phase. There are three forms of importing supported today: /path/to/file -Will import a file or directory from the local filesystem. If the file or -directory changes between stacker builds, it will be hashed and the new file -will be imported on subsequent builds. +Will import a file or directory from the local filesystem. If the file or directory changes between stacker builds, it will be hashed and the new file will be imported on subsequent builds. http://example.com/foo.tar.gz -Will import foo.tar.gz and make it available in `/stacker`. Note that stacker -will NOT update this file unless the cache is cleared, to avoid excess network -usage. That means that updates after the first time stacker downloads the file -will not be reflected. +Will import foo.tar.gz and make it available in `/stacker`. Note that stacker will NOT update this file unless the cache is cleared, to avoid excess network usage. That means that updates after the first time stacker downloads the file will not be reflected. stacker://$name/path/to/file @@ -88,10 +81,9 @@ Will grab /path/to/file from the previously built layer `$name`. #### `import hash` -The `import` directive also supports specifying the hash(sha256sum) of import source, -for all the three forms presented above, for example: +Each entry in the `import` directive also supports specifying the hash (sha256sum) of import source, for all the three forms presented above. For example: ``` -import: +imports: - path: config.json hash: f55af805b012017bc.... - path: http://example.com/foo.tar.gz @@ -99,14 +91,14 @@ import: - path: stacker://$name/path/to/file hash: f805b012017bc769a.... ``` -Before copying the file it will check if the requested hash matches the actual one. +Before copying the file, stacker will check that the file's hash matches the given value. For file imports, the source file is hashed at build time. For HTTP imports, the value returned by the server in the `X-Checksum-Sha256` HTTP header is checked first. If that value matches, the file is downloaded and then hashed and compared again. -`stacker build` supports the flag `--require-flag` which checks that all http(s) remote -imports have an hash in all stacker YAMLs. +`stacker build` supports the flag `--require-hash`, which will cause a build error if any HTTP(S) remote imports do not have a hash specified, in all transitively included stacker YAMLs. + +If `--require-hash` is not passed, this import mode can be combined with unchecked imports, and only files that have the hash specified will be checked. -This new import mode can be combined with the old one, for example: ``` -import: +imports: - path: "config.json hash: "BEEFcafeaaaaAAAA...." - /path/to/file @@ -116,13 +108,17 @@ import: The `import` directive also supports specifying the destination path (specified by `dest`) in the resulting container image, where the source file (specified -by `path`) will be copyied to, for example: +by `path`) will be copyed to. For example: ``` -import: +imports: - path: config.json dest: / ``` +### `import` (Deprecated) + +The deprecated `import` directive operates like `imports` except that the entries in the `import` array will be placed into `/stacker` instead of `/stacker/imports`. + ### `overlay_dirs` This directive works only with OverlayFS backend storage. @@ -146,7 +142,7 @@ to be available under container's /usr/local/ and all the files/dirs from the ho These correspond exactly to the similarly named bits in the [OCI image config spec](https://github.com/opencontainers/image-spec/blob/master/config.md#properties), -and are available for users to pass things through to the runtime environment +and are available for users to pass variables through to the runtime environment of the image. ### `generate_labels` @@ -163,19 +159,17 @@ be the label content. By default, environment variables do not pass through (pollute) the build environment. -`build_env`: this is a dictionary with environment variable definitions. - their values will be present in the build's environment. +`build_env`: A dictionary of environment variable definitions. The values will be present in the build's environment. -`build_env_passthrough`: This is a list of regular expressions that work as a +`build_env_passthrough`: A list of regular expressions that work as a filter on which environment variables should be passed through from the current -env into the container. To let all variables through simply set +env into the container. To let all variables through, simply set `build_env_passthrough`: `[".*"]` -If `build_env_passthrough` is not set, then the default value is to allow -through proxy variables `HTTP_PROXY, HTTPS_PROXY, FTP_PROXY, http_proxy, -https_proxy, ftp_proxy`. +If `build_env_passthrough` is not set, the default behavior is to allow +through proxy variables `HTTP_PROXY, HTTPS_PROXY, FTP_PROXY, http_proxy, https_proxy, ftp_proxy`. -Values in the `build_env` override values passed through via +Values in the `build_env` override values passed through via `build_env_passthrough`. ### `full_command` @@ -199,17 +193,17 @@ not include all of its build dependencies. - /foo/bar -> /bar/baz - /zomg -The first one binds /foo/bar to /bar/baz, and the second host /zomg to +The first binds /foo/bar to /bar/baz, and the second binds host /zomg to container /zomg. -Right now there is no awareness of change for any of these bind mounts, so ---no-cache should be used to re-build if the content of the bind mount has +At this time there is no awareness of change for any of these bind mounts, so +`--no-cache` should be used to re-build if the content of the bind mount has changed. ### `config` -`config` key is a special type of entry in the root in the `stacker.yaml` file. -It cannot contain a layer definition, it is used to provide configuration +The `config` key is a special type of entry in the root of the `stacker.yaml` file. +It cannot contain a layer definition; it is used to provide configuration applicable for building all the layers defined in this file. For example, config: @@ -221,16 +215,16 @@ applicable for building all the layers defined in this file. For example, If the `prerequisites` list is present under the `config` key, stacker will make sure to build all the layers in the stacker.yaml files found at the paths -contained in the list. This way stacker supports building multiple +contained in the list. In this way, stacker supports building multiple stacker.yaml files in the correct order. -In this particular case the parent folder of the current folder, let's call it -`parent`, has 3 subfolders `folder1`, `folder2` and `folder3`, each containing a +In this particular case, the parent folder of the current folder, let's call it +`parent`, has 3 subfolders (`folder1`, `folder2`, and `folder3`), each containing a `stacker.yaml` file. The example `config` above is in `parent/folder1/stacker.yaml`. -When `stacker build -f parent/folder1/stacker.yaml` is invoked, stacker would search -for the other two stacker.yaml files and build them first, before building -the stacker.yaml specified in the command line. +When `stacker build -f parent/folder1/stacker.yaml` is invoked, stacker searches +for the other two stacker.yaml files and builds them first, before building +the stacker.yaml file specified in the command line. #### `annotations` diff --git a/docs/user_guide/generate_sbom.md b/docs/user_guide/generate_sbom.md new file mode 100644 index 0000000..cadc553 --- /dev/null +++ b/docs/user_guide/generate_sbom.md @@ -0,0 +1,177 @@ +# Software Provenance Workflow Using SBOMs as OCI Artifacts + +!!! info + This article demonstrates a workflow for building an image while simultaneously creating a Software Bill of Materials (SBOM) as a related OCI artifact. + +## About SBOMs for container images + +The recent [Executive Order on Improving the Nation's Cybersecurity](https://www.whitehouse.gov/briefing-room/presidential-actions/2021/05/12/executive-order-on-improving-the-nations-cybersecurity/) provides guidance for hardening the software supply chain, including "providing a purchaser a Software Bill of Materials (SBOM) for each product directly or by publishing it on a public website." + +An SBOM is a comprehensive inventory of software components used in building a software image. The SBOM includes descriptions of all components, including libraries and modules, their relationships and hierarchies, and software supply chain origin information for each and every component of the image. The transparency provided by a detailed SBOM gives the user greater confidence in the security of the image and simplifies vulnerability assessment. + +You can create an SBOM for an existing container image by using commonly-available third-party tools to scan the image and generate the SBOM. However, as developers work to reduce the size of images, in some cases there may not be enough data or context to produce a correct SBOM. A better approach is to create the SBOM as part of the original build process. Stacker provides this capability. + + +## Workflow + +The following sections describe a step-by-step workflow for creating a stacker YAML file, then building and publishing an alpine container image that includes the SBOM. + +This example workflow implements the following scenario: + +- The stacker YAML file is named `b.yaml` +- The workflow builds an alpine container image +- The image is pushed to an OCI registry that is assumed to be running at `localhost:8080` + +The first three steps in the workflow modify the stacker YAML file. To view the full stacker file used in this example workflow, see [Reference: Full stacker file](#full-yaml). + +### Step 1: stacker file: Enable SBOM creation + +The build-time creation of an SBOM is enabled by including the `bom` attribute in the stacker YAML file (with `generate: true`), along with `annotations`. Both `bom` and `annotations` are required. This excerpt shows an example of the necessary sections and settings: + + bom-test: + from: + type: docker + url: docker://alpine:edge + bom: + generate: true + annotations: + org.opencontainers.image.authors: bom-test + org.opencontainers.image.vendor: bom-test + org.opencontainers.image.licenses: MIT + +### Step 2: stacker file: Discover all installed packages + +To assist in creating the SBOM, stacker includes a tool for discovering the packages that will be present in the image. Include the following `run` command in your stacker YAML file: + + run: | + /stacker/tools/static-stacker bom discover + + +### Step 3: stacker file: Enable cleanup after the build (optional) + +When the build is complete, you can remove unnecessary files to reduce the image size, as in this example for an alpine build: + + run: | + <...other commands...> + rm -f /etc/alpine-release /etc/apk/arch /etc/apk/repositories /etc/apk/world /etc/issue /etc/os-release /etc/secfixes.d/alpine /lib/apk/db/installed /lib/apk/db/lock /lib/apk/db/scripts.tar /lib/apk/db/triggers + + +### Step 4: Build the image + +Run the `stacker build` command, using the modified stacker file. + + $ stacker build -f b.yaml + + +### Step 5: Publish the image to the OCI registry + +Run the `stacker publish` command to publish the container image to the OCI registry. + + $ ./stacker publish \ + --stacker-file b.yaml \ + --tag latest \ + --url docker://localhost:8080/ \ + --skip-tls + + +### Step 6: Inspect the artifact tree + +!!! info + Changes added in [OCI Distribution Spec v1.1.0-rc3](https://github.com/opencontainers/distribution-spec/releases/tag/v1.1.0-rc3) and [OCI Image Spec v1.1.0-rc4](https://github.com/opencontainers/image-spec/releases/tag/v1.1.0-rc4) (summarized [here](https://opencontainers.org/posts/blog/2023-07-07-summary-of-upcoming-changes-in-oci-image-and-distribution-specs-v-1-1/)) allow arbitrary artifact types and references. These changes support software supply chain use cases such as SBOMs. + +Run `regctl artifact tree` to display the artifact tree of the container image in the registry. The artifact tree for this image includes two SBOM-related artifact files. + + $ ./regctl artifact tree localhost:8080/bom-test:latest + + Ref: localhost:8080/bom-test:latest + Digest: sha256:9172c5f692f2c65e4f773448503b21dba2de6454bd159905c4bf6d83176e4ea3 + Referrers: + - sha256:9c0655368b10ca4b2ffe39e4dd261fb89df25a46ae92d6eb4e6e1792a451883e: application/spdx+json + - sha256:9df25a46ae92d6eb4e6e1792a451883e9c0655368b10ca4b2ffe39e4dd261fb8: application/vnd.stackerbuild.inventory+json + + + +The displayed artifact tree shows that the original image (`localhost:8080/bom-test:latest`) contains two direct referrers: + +- `application/spdx+json` is a comprehensive [SPDX](https://spdx.dev/) SBOM file in JSON format +- `application/vnd.stackerbuild.inventory+json` is a summary inventory, listing the file name, file size, SHA digest, and mode of every file in the SBOM + +The first two entries in the inventory file are shown here: + + $ cat .stacker/artifacts/bom-test/inventory.json | jq . | less + + { + { + "path": "/bin/arch", + "size": 12, + "checksum": "", + "mode": "01000000777" + }, + { + "path": "/bin/ash", + "size": 12, + "checksum": "", + "mode": "01000000777" + }, + ... + + +## Adding additional files to a package and SBOM + + You can add to the image one or more files or directories that are not otherwise included in the build. This capability is useful if, for example, you are building a well-known distribution but you want to include some other content in the image. If the extra content is added after the `static-stacker bom discover` tool in the `run` section of the stacker file, you must explicitly provide the SBOM information in the `bom` section. Otherwise the SBOM creation will due to missing SBOM information, causing the build to fail. + + Use the `bom/packages` attribute in the stacker YAML file to provide SBOM information for the additional files or directory paths. In the following example, the new content is added as a package. A name (`pkg1`), version, and license information is provided for the package. Under the `paths` attribute, provide a bracketed, comma-separated list of the files or directories to add. In this example, a single file named `file1` is added. + + bom: + generate: true + packages: + - name: pkg1 + version: 1.0.0 + license: Apache-2.0 + paths: [/file1] + +With the SBOM information provided, you can now safely use `file1` in the `run` section, as shown here: + + run: | + /stacker/tools/static-stacker bom discover + touch /file1 + + + + + +## Reference: Full stacker file + +> :bulb: To copy the file to the clipboard, click the copy icon that appears in the upper right corner of the expanded text box. + +
+ Click here to view the complete stacker file for this workflow example + +```shell +bom-test: + from: + type: docker + url: docker://alpine:edge + bom: + generate: true + packages: + - name: pkg1 + version: 1.0.0 + license: Apache-2.0 + paths: [/file1] + annotations: + org.opencontainers.image.authors: bom-test + org.opencontainers.image.vendor: bom-test + org.opencontainers.image.licenses: MIT + run: | + # discover installed pkgs + /stacker/tools/static-stacker bom discover + # run our cmds + ls -al / + # some changes + touch /file1 + # cleanup + rm -f /etc/alpine-release /etc/apk/arch /etc/apk/repositories /etc/apk/world /etc/issue /etc/os-release /etc/secfixes.d/alpine /lib/apk/db/installed /lib/apk/db/lock /lib/apk/db/scripts.tar /lib/apk/db/triggers +``` + +
diff --git a/docs/whats-new.md b/docs/whats-new.md index 5aec019..f3a741c 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -1,43 +1,62 @@ # What's New -## [v1.0.0](https://github.com/project-stacker/stacker/releases/tag/v1.0.0-rc4) +## [v1.0.0](https://github.com/project-stacker/stacker/releases/tag/v1.0.0-rc9) -* Convert a Dockerfile for stacker +### Convert a Dockerfile for stacker - A new [`stacker convert`](reference/stacker_cli.md#stacker-convert) command performs a conversion of a Dockerfile into a stacker.yaml file. During the conversion, some variables from the Dockerfile may be exported to a substitution file. +- A new [`stacker convert`](reference/stacker_cli.md#stacker-convert) command performs a conversion of a Dockerfile into a stacker.yaml file. During the conversion, some variables from the Dockerfile may be exported to a substitution file. - > **NOTE:** The conversion is a best-effort process and may not be successful in all cases. + :pencil2: The conversion is a best-effort process and may not be successful in all cases. +### Publish specific layers -* Publish specific layers +- By default, the [`stacker publish`](reference/stacker_cli.md#stacker-publish) command pushes all layers in a stacker.yaml file instead of only the required layers. Using a new command option, `layer `, you can explicitly specify which layers are to be published. This command option can be specified multiple times, selecting each layer to be included. - By default, the [`stacker publish`](reference/stacker_cli.md#stacker-publish) command pushes all layers in a stacker.yaml file instead of only the required layers. Using a new command option, `layer `, you can explicitly specify which layers are to be published. This command option can be specified multiple times, selecting each layer to be included. +### Specify a single working directory -* Specify a single working directory +- A new [`stacker`](reference/stacker_cli.md#stacker) command option, `work-dir`, sets the working directory for stacker's cache, OCI output, and rootfs output. The existing command options `stacker-dir`, `oci-dir`, and `roots-dir` can then be omitted or used to override the `work-dir` setting. - A new [`stacker`](reference/stacker_cli.md#stacker) command option, `work-dir`, sets the working directory for stacker's cache, OCI output, and rootfs output. The existing command options `stacker-dir`, `oci-dir`, and `roots-dir` can then be omitted or used to override the `work-dir` setting. +### Import contents when no shell exists +- Import directives can include destination paths. This feature is useful when images are built from a scratch layer before any shell has been invoked. Prior to this, a `run:` section was required to invoke a shell and to explicitly copy files to be imported into the image. For example, you can now write a directive such as the following, with no `run:` section: -* Report kernel version and fs type + test: + from: + type: scratch + imports: + - path: test_file + dest: /files/ + - path: test_file2 + dest: /file2 - The [`stacker check`](reference/stacker_cli.md#stacker-check) command now reports this information. +### Generate SBOMs during the build -* Build improvements +- Changes added in [OCI Distribution Spec v1.1.0-rc3](https://github.com/opencontainers/distribution-spec/releases/tag/v1.1.0-rc3) and [OCI Image Spec v1.1.0-rc4](https://github.com/opencontainers/image-spec/releases/tag/v1.1.0-rc4) (summarized [here](https://opencontainers.org/posts/blog/2023-07-07-summary-of-upcoming-changes-in-oci-image-and-distribution-specs-v-1-1/)) allow arbitrary artifact types and references. These changes support software supply chain use cases such as SBOMs. +- For a demonstration of an OCI artifacts workflow that generates an SBOM, see [Software Provenance Workflow Using OCI Artifacts](user_guide/generate_sbom.md). + +### Report kernel version and fs type + +- The [`stacker check`](reference/stacker_cli.md#stacker-check) command now reports this information. + +### Build improvements + +*** ## [v0.40.1](https://github.com/project-stacker/stacker/releases/tag/v0.40.1) -* Support for `scratch` - Prior to v0.40.1, `stacker` did not support empty root filesystems to be used a base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host statically built binaries. +### Support for `scratch` + +- Prior to v0.40.1, `stacker` did not support empty root filesystems to be used a base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host statically built binaries. -* Support for `import`ing content into container image +### Support for `import`ing content into container image - Prior to v0.40.1, copying content into a layer permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. +- Prior to v0.40.1, copying content into a layer permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. -* Publish with substitutions specified in a file +### Publish with substitutions specified in a file - Using a new `build` command option, [`substitute-file `](reference/stacker_cli.md#stacker-build), you can now declare variable substitutions in a file instead of the command line. The substitution file uses a 'FOO: bar' key-value yaml format to declare substitutions. +- Using a new `build` command option, [`substitute-file `](reference/stacker_cli.md#stacker-build), you can now declare variable substitutions in a file instead of the command line. The substitution file uses a 'FOO: bar' key-value yaml format to declare substitutions. -* Some `squashfs` improvements +### Some `squashfs` improvements - While building squashfs layers, use `squashfuse_ll` if available which is faster. +- While building squashfs layers, use `squashfuse_ll` if available which is faster. diff --git a/mkdocs.yml b/mkdocs.yml index 81474bf..64ad122 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -120,6 +120,7 @@ nav: - Build Cache: user_guide/build_cache.md - Inspect Images: user_guide/inspect_images.md - Image Chroot: user_guide/image_chroot.md + - Software Provenance Using SBOMs: user_guide/generate_sbom.md - Grab Image Content: user_guide/grab_image_content.md - Debugging: user_guide/debugging.md - Converting a Dockerfile: user_guide/convert_dockerfile.md From 6f64e020ce922cbfd1ba8a166dcb8b8ec66584e5 Mon Sep 17 00:00:00 2001 From: mbshields Date: Wed, 10 Jan 2024 17:10:46 -0800 Subject: [PATCH 3/4] docs: resolve more PR comments Signed-off-by: mbshields --- docs/reference/stacker_cli.md | 2 +- docs/reference/stacker_file.md | 6 +++--- docs/user_guide/template_substitution.md | 8 +++++++- docs/whats-new.md | 8 ++++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/reference/stacker_cli.md b/docs/reference/stacker_cli.md index a153fe4..e1358a5 100644 --- a/docs/reference/stacker_cli.md +++ b/docs/reference/stacker_cli.md @@ -29,7 +29,7 @@ This document lists the command line interface (CLI) commands for stacker versio stacker [global options] command [command options] [arguments...] VERSION: - stacker v1.0.0-rc4-8e267fc liblxc 727adc05220e9b8b594c5bc2124bef5b2e0abe7a + stacker liblxc COMMANDS: build builds a new OCI image from a stacker yaml file diff --git a/docs/reference/stacker_file.md b/docs/reference/stacker_file.md index 7009a87..e8c84b4 100644 --- a/docs/reference/stacker_file.md +++ b/docs/reference/stacker_file.md @@ -203,8 +203,8 @@ changed. ### `config` The `config` key is a special type of entry in the root of the `stacker.yaml` file. -It cannot contain a layer definition; it is used to provide configuration -applicable for building all the layers defined in this file. For example, +It cannot contain an image definition; it is used to provide configuration +applicable for building all the images defined in this file. For example, config: prerequisites: @@ -214,7 +214,7 @@ applicable for building all the layers defined in this file. For example, #### `prerequisites` If the `prerequisites` list is present under the `config` key, stacker will -make sure to build all the layers in the stacker.yaml files found at the paths +make sure to build all the images in the stacker.yaml files found at the paths contained in the list. In this way, stacker supports building multiple stacker.yaml files in the correct order. diff --git a/docs/user_guide/template_substitution.md b/docs/user_guide/template_substitution.md index ef817d4..21a990d 100644 --- a/docs/user_guide/template_substitution.md +++ b/docs/user_guide/template_substitution.md @@ -1,6 +1,6 @@ # Template Variable Substitution -When doing a `stacker build`, the behavior of stacker is specified by the yaml +When doing a `stacker build`, the behavior of stacker is specified by the YAML directives below. In addition to these, stacker allows variable substitions of several forms. For example, a line like: @@ -19,3 +19,9 @@ without a default; to make the default an empty string, use `${FOO:}`. You can also declare variable substitutions in a separate file which is then included in the build command as in this example: `stacker build --substitute-file ` + +The substitution file simply contains KEY:VALUE pairs, as in this example: + + $ cat stacker-subs.yaml + ONE: 1 + TWO: 2 diff --git a/docs/whats-new.md b/docs/whats-new.md index f3a741c..26884bd 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -4,7 +4,7 @@ ### Convert a Dockerfile for stacker -- A new [`stacker convert`](reference/stacker_cli.md#stacker-convert) command performs a conversion of a Dockerfile into a stacker.yaml file. During the conversion, some variables from the Dockerfile may be exported to a substitution file. +- A new [`stacker convert`](reference/stacker_cli.md#stacker-convert) command performs a conversion of a Dockerfile into a stacker.yaml file. During the conversion, some variables from the Dockerfile may be exported to a substitution file that can be included in `stacker build` using the `--substitute-file ` command option. :pencil2: The conversion is a best-effort process and may not be successful in all cases. @@ -14,9 +14,9 @@ ### Specify a single working directory -- A new [`stacker`](reference/stacker_cli.md#stacker) command option, `work-dir`, sets the working directory for stacker's cache, OCI output, and rootfs output. The existing command options `stacker-dir`, `oci-dir`, and `roots-dir` can then be omitted or used to override the `work-dir` setting. +- A new [`stacker`](reference/stacker_cli.md#stacker) command option, `--work-dir`, sets the working directory for stacker's cache, OCI output, and rootfs output. The existing command options `--stacker-dir`, `--oci-dir`, and `--roots-dir` can then be omitted or used to override the `--work-dir` setting. -### Import contents when no shell exists +### Import contents when no shell exists in the base image - Import directives can include destination paths. This feature is useful when images are built from a scratch layer before any shell has been invoked. Prior to this, a `run:` section was required to invoke a shell and to explicitly copy files to be imported into the image. For example, you can now write a directive such as the following, with no `run:` section: @@ -51,7 +51,7 @@ ### Support for `import`ing content into container image -- Prior to v0.40.1, copying content into a layer permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. +- Prior to v0.40.1, copying content into an image permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. ### Publish with substitutions specified in a file From 654b94831a8b5ef605559378e03996615f5e9a65 Mon Sep 17 00:00:00 2001 From: mbshields Date: Fri, 12 Jan 2024 14:20:34 -0800 Subject: [PATCH 4/4] docs: resolve comments, change zotregistry.io to .dev Signed-off-by: mbshields --- docs/reference/stacker_cli.md | 6 +++--- docs/reference/stacker_file.md | 2 +- docs/whats-new.md | 10 +++++----- material/overrides/home.html | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/reference/stacker_cli.md b/docs/reference/stacker_cli.md index e1358a5..041f5c2 100644 --- a/docs/reference/stacker_cli.md +++ b/docs/reference/stacker_cli.md @@ -5,13 +5,13 @@ This document lists the command line interface (CLI) commands for stacker versio - + - + @@ -146,7 +146,7 @@ This document lists the command line interface (CLI) commands for stacker versio --show-only show the images to be published without actually publishing them --force force publishing the images present in the OCI layout even if they should be rebuilt --layer-type value set the output layer type (supported values: tar, squashfs); can be supplied multiple times (default: "tar") - --layer value layer to be published; can be specified multiple times + --image value image to be published; can be specified multiple times diff --git a/docs/reference/stacker_file.md b/docs/reference/stacker_file.md index e8c84b4..3534672 100644 --- a/docs/reference/stacker_file.md +++ b/docs/reference/stacker_file.md @@ -77,7 +77,7 @@ Will import foo.tar.gz and make it available in `/stacker`. Note that stacker wi stacker://$name/path/to/file -Will grab /path/to/file from the previously built layer `$name`. +Will grab /path/to/file from the previously built image `$name`. #### `import hash` diff --git a/docs/whats-new.md b/docs/whats-new.md index 26884bd..d160fde 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -8,9 +8,9 @@ :pencil2: The conversion is a best-effort process and may not be successful in all cases. -### Publish specific layers +### Publish specific images -- By default, the [`stacker publish`](reference/stacker_cli.md#stacker-publish) command pushes all layers in a stacker.yaml file instead of only the required layers. Using a new command option, `layer `, you can explicitly specify which layers are to be published. This command option can be specified multiple times, selecting each layer to be included. +- By default, the [`stacker publish`](reference/stacker_cli.md#stacker-publish) command pushes all images in a stacker.yaml file instead of only the required images. Using a new command option, `--image `, you can explicitly specify which images are to be published. This command option can be specified multiple times, selecting each image to be included. ### Specify a single working directory @@ -18,7 +18,7 @@ ### Import contents when no shell exists in the base image -- Import directives can include destination paths. This feature is useful when images are built from a scratch layer before any shell has been invoked. Prior to this, a `run:` section was required to invoke a shell and to explicitly copy files to be imported into the image. For example, you can now write a directive such as the following, with no `run:` section: +- Import directives can include destination paths. This feature is useful to simplify `run` section scripts, and for when images are built without a base image. With no base image, there is no shell to run the script in a `run` section. Prior to this change, a `run:` section was required to invoke a shell and to explicitly copy files to be imported into the image. For example, you can now write a directive such as the following, with no `run:` section: test: from: @@ -47,11 +47,11 @@ ### Support for `scratch` -- Prior to v0.40.1, `stacker` did not support empty root filesystems to be used a base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host statically built binaries. +- Prior to v0.40.1, `stacker` did not support empty root filesystems to be used as a base container image. The support has now been [added](reference/stacker_file.md#from) which can be used to host statically built binaries. ### Support for `import`ing content into container image -- Prior to v0.40.1, copying content into an image permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. +- Prior to v0.40.1, copying content into a scratch image permanently involved bind mounting a shell such as busybox and invoking appropriate commands using the `run` directive. Now the `import` directive [allows](reference/stacker_file.md#import-dest) for the `dest` option to achieve the same. ### Publish with substitutions specified in a file diff --git a/material/overrides/home.html b/material/overrides/home.html index 1db1175..300c623 100644 --- a/material/overrides/home.html +++ b/material/overrides/home.html @@ -36,7 +36,7 @@

Stacker Build

User Guide -

Pair stacker with zot, an OCI-native Image Registry

+

Pair stacker with zot, an OCI-native Image Registry

stackerStacker builds OCI images
stacker buildBuilds a new OCI image from a stacker yaml file
stacker recursive-buildFinds stacker yaml files under a directory and builds all OCI layers they define
stacker recursive-buildFinds stacker yaml files under a directory and builds all OCI images they define
stacker convertConverts a Dockerfile into a stacker yaml file
stacker publishPublishes OCI images previously built from one or more stacker yaml files
stacker chrootRuns a command in a chroot (same as stacker exec)
stacker cleanCleans up after a stacker build
stacker inspectPrints the json representation of an OCI image
stacker grabGrabs a file from the layer's filesystem
stacker grabGrabs a file from the image's filesystem
stacker unpriv-setupDoes the necessary unprivileged setup for stacker build to work without root
stacker gcGarbage collection of unused OCI imports/outputs snapshots
stacker checkChecks that all runtime required items (like kernel features) are present