From 84e8e6f5e103cc3cbf7057dbdf5deca819d8f3f8 Mon Sep 17 00:00:00 2001 From: Simon de Vlieger Date: Thu, 14 Nov 2024 08:31:05 +0100 Subject: [PATCH] project: import `otk` Automatically import `otk`'s `doc/` subdirectory into the project pages. Signed-off-by: Simon de Vlieger --- .github/workflows/pull.yml | 5 + Makefile | 4 + .../02-projects/otk/00-installation.md | 24 ++ .../02-projects/otk/01-contributing.md | 9 + .../02-projects/otk/02-usage.md | 10 + .../otk/03-omnifest/01-directive.md | 194 ++++++++++ .../otk/03-omnifest/02-external.md | 338 ++++++++++++++++++ .../02-projects/otk/03-omnifest/index.md | 42 +++ .../02-projects/otk/04-best-practices.md | 11 + .../02-projects/otk/05-deprecation.md | 10 + .../02-projects/otk/06-security.md | 3 + .../02-projects/otk/07-integration.md | 3 + docs/developer-guide/02-projects/otk/index.md | 5 + scripts/pull_otk.py | 44 +++ 14 files changed, 702 insertions(+) create mode 100644 docs/developer-guide/02-projects/otk/00-installation.md create mode 100644 docs/developer-guide/02-projects/otk/01-contributing.md create mode 100644 docs/developer-guide/02-projects/otk/02-usage.md create mode 100644 docs/developer-guide/02-projects/otk/03-omnifest/01-directive.md create mode 100644 docs/developer-guide/02-projects/otk/03-omnifest/02-external.md create mode 100644 docs/developer-guide/02-projects/otk/03-omnifest/index.md create mode 100644 docs/developer-guide/02-projects/otk/04-best-practices.md create mode 100644 docs/developer-guide/02-projects/otk/05-deprecation.md create mode 100644 docs/developer-guide/02-projects/otk/06-security.md create mode 100644 docs/developer-guide/02-projects/otk/07-integration.md create mode 100644 docs/developer-guide/02-projects/otk/index.md create mode 100755 scripts/pull_otk.py diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index e858b8478..9d54d4a79 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -32,6 +32,11 @@ jobs: run: | make pull-osbuild-modules + - name: Pull otk + working-directory: ./osbuild.github.io + run: | + make pull-otk + - name: Setup node uses: actions/setup-node@v4 with: diff --git a/Makefile b/Makefile index 4c60e8cd7..2ec107914 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ test: ## test pulling the readmes from the other projects pull-readmes: ## pull the readmes from other projects given in `readme-list` python3 scripts/pull_readmes.py readme-list +.PHONY: pull-otk +pull-otk: ## pull the otk documentation + python3 scripts/pull_otk.py + .PHONY: pull-osbuild-modules pull-osbuild-modules: ## pull the documentation of the osbuild modules python3 scripts/pull_osbuild_modules.py diff --git a/docs/developer-guide/02-projects/otk/00-installation.md b/docs/developer-guide/02-projects/otk/00-installation.md new file mode 100644 index 000000000..58cbecbee --- /dev/null +++ b/docs/developer-guide/02-projects/otk/00-installation.md @@ -0,0 +1,24 @@ +# Installation + +As `otk` is still in a proof of concept state it is not yet packaged for any distributions. Installation thus requires you to work from [source](https://github.com/osbuild/otk). + +To start hacking on `otk` you can: + +``` +€ git clone https://github.com/osbuild/otk +# ... +€ python3 -m venv venv +# ... +€ . venv/bin/activate +# ... +€ pip install -e ".[dev]" +# ... +€ make external +# ... +€ pre-commit install +# ... +``` + +This will get you an activated Python virtual environment with an editable install of `otk`. You can then run edit source in `src/` and run `otk` as long as your virtual environment is enabled. When you do a `git commit` some verification steps will run locally on the changed files. + +You can read more about [contributing](./01-contributing.md) diff --git a/docs/developer-guide/02-projects/otk/01-contributing.md b/docs/developer-guide/02-projects/otk/01-contributing.md new file mode 100644 index 000000000..3f890c483 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/01-contributing.md @@ -0,0 +1,9 @@ +# Contributing + +`otk` is written in Python with a minimal version of `3.9`. It is licensed under the Apache License. To get started with contributing to `otk` [check out the repository and install the dependencies](./00-installation.md). After you've done so familiarize yourself a bit with the layout of the source code. + +You can run the test suite with `make test` or `pytest` in your source checkout directory. + +## What to do? + +You can find open [issues](https://github.com/osbuild/otk/issues) at our [GitHub](https://github.com/osbuild/otk) page. diff --git a/docs/developer-guide/02-projects/otk/02-usage.md b/docs/developer-guide/02-projects/otk/02-usage.md new file mode 100644 index 000000000..d1e96a125 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/02-usage.md @@ -0,0 +1,10 @@ +# Usage + +After [installing](./00-installation.md) `otk` you'll probably want to start using it. If you've followed the [installation instructions](./00-installation.md) for a source checkout you'll have an examples directory available. You can compile one of the examples like so: + +``` +€ OTK_EXTERNAL_PATH="./external" otk compile example/centos/centos-9-x86_64-minimal-raw.yaml +# ... +``` + +Note that this example will take some time to generate, this is due to dependency solving. After the command is done it will output the generated content on `STDOUT`. diff --git a/docs/developer-guide/02-projects/otk/03-omnifest/01-directive.md b/docs/developer-guide/02-projects/otk/03-omnifest/01-directive.md new file mode 100644 index 000000000..c78983cb8 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/03-omnifest/01-directive.md @@ -0,0 +1,194 @@ +# Directive + +In [omnifests](./index.md) directives are sections of the document that get transformed by `otk` into something else. + +`otk` has various directives that can be used in an omnifest. Generally these directives can appear anywhere in the tree unless otherwise specified (see below) and they are replaced with other trees or values as produced by the directive. + +There are [example omnifests](https://github.com/osbuild/otk/tree/main/example) for various distributions and images in a [source checkout](../00-installation.md). + +## `otk.version` + +### Example + +```yaml +otk.version: "1" +``` + +## `otk.target..` + +Only act on this sub-tree if producing output for the specified consumer. Anything specific to the pipelines of e.g. osbuild would be put under `otk.target.osbuild`. This allows `otk` to infer context for the omnifest that is being processed. + +A target is necessary for `otk` to generate any outputs. The target is namespaced to a specific application. `otk` tries to keep little context but it does need to know what it is outputting for. This allows us to scope [otk.external](#otkexternal) things to only be allowed within specific targets and for those externals to assume certain things will be in the tree. + +The following values are valid for the `` part of the key, this list can grow as other image build tooling is supported: + +- `osbuild` + +The `` part of the key is free form and allows you to use a descriptive name for the export. Note that there MUST be no duplication of the `.` tuple. + +```yaml +otk.target.osbuild.tree: + pipelines: + - otk.include: pipelines/root.yaml + - otk.include: pipelines/tree.yaml +``` + +--- + +## `otk.define` + +Defines variables that can be used through the `${}` directive in +other parts of the omnifest. + +Variable scope is global, an `otk.define` directive anywhere in the omnifest +tree will result in the defined names being hoisted to the global scope. + +Redefinitions of variables are allowed. This allows for setting up default +values. If `-W duplicate-definition` is passed as an argument to `otk` then +`otk` will warn on all duplicate definitions. + +Expects a `map` for its value. + +```yaml +otk.define: + packages: + include: + - @core + - kernel + exclude: + - linux-util + boot_mode: uefi +``` + +Valid variable names must start with `[a-zA-Z]` and after that initial +char can also contain `[a-zA-Z0-9_]`. E.g. `foo` is valid but `f?` is +not. + +## Usage of `${}` + +Use a previously defined variable. String values can be used inside other +string values, non-string values *must* stand on their own. + +```yaml +otk.define: + variable: "foo" + +otk.include: ${variable} +``` + +Using the above `packages` map example you can refer to the include and exclude +lists using `${packages.include}` and `${packages.exclude}`. + +If a `${}` appears in a `str` value then its string value as it appears +in `otk.define` is replaced into the string. Note that substitutions +in this form require the value to be a string in the `otk.define`. + +```yaml +# this is OK +otk.define: + variable: aarch64 + +otk.include: path/${variable}.yaml +``` + +The following example is an error as the value of `variable` is a `seq`, which +is not allowed inside a string format. + +```yaml +# this is NOT OK +otk.define: + variable: + - 1 + - 2 + +otk.include: path/${variable}.yaml +``` + +This is okay because `${variable}` is there on it's own so it's unambiguous. +```yaml +# this is OK +otk.define: + variable: + - 1 + - 2 + +some: + thing: ${variable} +``` + +## `otk.include` + +Include a file at this position in the tree, replacing the directive with the +contents of the file. + +Note that cyclical includes are forbidden and will cause an error. + +It expects a `str` for its value and as with other strings variable substitution +is performed before using it. + +```yaml +otk.include: file.yaml +``` + +## `otk.op` + +Perform various operations on variables. + +### `otk.op.join` + +Join two or more variables of type `sequence` or `map` together, trying to +join other types or mix types will cause an error. Duplicate keys in +maps are considered an error. + +Expects a `map` for its value that contains a `values` key with a value of type +`seq` or `map`. + +Example when using with a `sequence` as input: + +```yaml +otk.define: + a: + - 1 + - 2 + b: + - 3 + - 4 + c: + otk.op.join: + values: + - ${a} + - ${b} + +-> Result c: [1, 2, 3, 4] +``` + +Example when using with a `map` as input: + +```yaml +otk.define: + a: + a: 1 + b: + b: 2 + c: + otk.op.join: + values: + - ${a} + - ${b} + +-> Result + c: + a: 1 + b: 2 +``` + +## `otk.external` + +External directives. Directives starting with `otk.external` are redirected +to `/usr/libexec/otk/external`-binaries. For example the directive +`otk.external.osbuild-depsolve-dnf4` will execute `osbuild-depsolve-dnf4` +with the tree under the directive on stdin and expect a new tree to replace +the directive with on stdout. + +Read more about [external directives](./02-external.md) in their specific +documentation section. diff --git a/docs/developer-guide/02-projects/otk/03-omnifest/02-external.md b/docs/developer-guide/02-projects/otk/03-omnifest/02-external.md new file mode 100644 index 000000000..a863df034 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/03-omnifest/02-external.md @@ -0,0 +1,338 @@ +# External + +External directives are directives that are implemented externally from `otk`. +They are meant to be used to provide target-specific behavior and can be used +to express things where `otk` is not expressive enough. + +## Protocol + +Directives are small executables that receive JSON and are expected to output +JSON again in a specific format. Additional keys are not allowed. + +When we have an [omnifest](./index.md) that looks like this: + +```yaml +otk.external.name: + child: + - 1 + options: "are here" +``` + +The external gets called with the following JSON: + +```json +{ + "tree": { + "child": [1], + "options": "are here" + } +} +``` + +And is expected to return a JSON dict with a single top-level `tree` object that can contain arbitrary values. Additional top-level keys are not allowed. Example: + +```json +{ + "tree": { + "what": { + "you": "want" + } + } +} +``` + +Which will replace the previous `otk.external.name` subtree with the output of new subtree from the external command. Example: + +```yaml +tree: + what: + you: "want" +``` + +If the external returns `{}`, an empty object, `otk` will assume that there is +no tree to replace and remove the node instead. This will turn the following +YAML: + +```yaml +dummy: + otk.external.name: + options: +``` + +Into this YAML structure: + +```yaml +dummy: +``` + +## Example + +The following example demonstrates how an external can be used to implement string +concatenation. + +Given the following script called `concat` in `/usr/local/libexec/otk/concat`: + +```bash +#!/usr/bin/env bash + +output="$(cat - | jq -jr '.tree.parts[]')" +echo "{\"tree\":{\"output\":\"$output\"}}" +``` + +and the following directive: + +```yaml +examplestring: + otk.external.concat: + parts: + - list + - of + - strings +``` + +the script is called with the following data on stdin: + +```json +{ + "tree": { + "parts": [ + "list", + "of", + "strings" + ] + } +} +``` + +which results in the following output: + +```json +{ + "tree": { + "output": "listofstrings" + } +} +``` + +and the final omnifest will be: + +```yaml +examplestring: + output: listofstrings +``` + +## Paths + +`otk` will look for external directives in the following paths, stopping when +it finds the first match: + +- A path defined by the `OTK_EXTERNAL_PATH` environment variable. +- `/usr/local/libexec/otk/external` +- `/usr/libexec/otk/external` +- `/usr/local/lib/otk/external` +- `/usr/lib/otk/external` + +The filename for an external executable is based on the external name. When the +following directive is encountered: `otk.external.` then +`otk` will try to find an executable called `` in the previously +mentioned search paths. + +Examples: + +- `otk.external.foo` -> `foo` +- `otk.external.osbuild-bar` -> `osbuild-bar` + +## Naming + +So far we've followed a common naming scheme for the externals we're providing. +So far externals usually generate defines which are then later used by other +externals to transform those variables into (for example) `osbuild` stages or +sources. + +For our naming we prefix the externals that generate variables in a define block +with: `osbuild-gen-` which indicates that this external is meant for use with +`osbuild` and that it *generates* variables. + +Externals that use (generated) variables follow the idiom of `osbuild-make-`. + +As an example the `osbuild-gen-depsolve-dnf4` external will dependency solve its +options into a bunch of variables which are then later used to generate stages in +an `osbuild` target with `osbuild-make-depsolve-dnf4-rpm-stage` and sources with +`osbuild-make-depsolve-dnf4-curl-source`. + +## Implementations + +`otk` currently ships together with some external implementations to work with +`osbuild`. + +### `osbuild-gen-depsolve-dnf4`. + +The `osbuild` target requires a list of solved packages and sources in its manifest. +The `osbuild-gen-depsolve-dnf4` external is used to generate these based on a list +of packages to be included and excluded. + +This external requires `osbuild-depsolve-dnf` to be installed on the system that runs +`otk`. + +An example invocation is like this: + +```yaml +otk.define: + packages: + otk.external.osbuild-gen-depsolve-dnf4: + architecture: "x86_64" + module_platform_id: "c9s" + releasever: "9" + repositories: + - id: "foo" + baseurl: "https://example.com/" + - id: "bar" + baseurl: "https://example.com/" + packages: + include: + - "@core" + exclude: + - "foo" +``` + +The result in the `packages` variable can then be used by other externals. + +### `osbuild-make-depsolve-dnf4-rpm-stage` + +Use the generated defines from `osbuild-gen-depsolve-dnf4` to create an RPM +stage for an `osbuild` manifest: + +```yaml +otk.external.osbuild-make-depsolve-dnf4-rpm-stage: + packageset: ${packages} # generated by `osbuild-gen-depsolve-dnf4` + gpgkeys: + - "---" + - "---" +``` + +### `osbuild-make-depsolve-dnf4-curl-source` + +Use the generated defines from `osbuild-gen-depsolve-dnf4` to create a sources +list for an `osbuild` manifest. + +```yaml +otk.target.osbuild: + sources: + otk.external.osbuild-make-depsolve-dnf4-curl-source: + packagesets: + - ${packages} # generated by `osbuild-gen-depsolve-dnf4` +``` + +### `osbuild-gen-partition-table` + +The `osbuild-gen-partition-table` generates defines for stages and mounts that +create filesystems and partition tables in `osbuild`. + +```yaml +otk.define: + filesystem: + otk.external.osbuild-gen-partition-table: + modifications: {} # empty + properties: + type: gpt + bios: true + default_size: "10 GiB" + uuid: D209C89E-EA5E-4FBD-B161-B461CCE297E0 + create: + bios_boot_partition: false + esp_partition: true + esp_partition_size: "200 MiB" + partitions: + - name: boot + mountpoint: /boot + label: boot + size: "1 GiB" + type: "xfs" + fs_mntops: defaults + # XXX: should we derive this automatically from the mountpoint? + part_type: BC13C2FF-59E6-4262-A352-B275FD6F7172 + # we use hardcoded uuids for compatibility with "images" + part_uuid: CB07C243-BC44-4717-853E-28852021225B + - name: root + mountpoint: / + label: root + type: "xfs" + size: "2 GiB" + fs_mntops: defaults + # XXX: should we derive this automatically from the mountpoint? + part_type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + # we use hardcoded uuids for compatibility with "images" + part_uuid: 6264D520-3FB9-423F-8AB8-7A0A8E3D3562 + # XXX: it would be nicer if the "fs_options" could be part of their + # stages directly without this indirection. + fs_options: + otk.external.osbuild-make-partition-mounts-devices: + ${filesystem} +``` + +### `osbuild-make-partition-mounts-devices` + +Creates mounts and devices for `osbuild` based on the +`osbuild-gen-partition-table` external. + +```yaml +otk.external.osbuild-make-partition-mounts-devices: + ${filesystem} # generated by `osbuild-gen-partition-table` +``` + +### `osbuild-make-partition-stages` + +Creates stages for `osbuild` based on the `osbuild-gen-partition-table` +external. + +The following example generates the stages and copy-to-device stages for +an image pipeline: + +```yaml +otk.op.join: + values: + - otk.external.osbuild-make-partition-stages: + ${filesystem} # generated by `osbuild-gen-partition-table` + - - type: org.osbuild.copy + inputs: + root-tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:os + options: + paths: + - from: input://root-tree/ + to: mount://-/ + devices: + ${fs_options.devices} # generated by `osbuild-gen-partition-table` + mounts: + ${fs_options.mounts} # generated by `osbuild-gen-partition-table` +``` + +### `osbuild-gen-inline-files` + +Generate variables for inline files so they can be used as `osbuild` sources: + +```yaml +files: + otk.external.osbuild-gen-inline-files: + inline: + kickstart: + contents: | + # Run initial-setup on first boot + # Created by osbuild + firstboot --reconfig + lang en_US.UTF-8 +``` + +### `osbuild-make-inline-source` + +Use the variables as generated by `osbuild-gen-inline-files` to create the +necessary sources for `osbuild`. + +```yaml +otk.external.osbuild-make-inline-source: + const: + files: ${files.const.files} +``` diff --git a/docs/developer-guide/02-projects/otk/03-omnifest/index.md b/docs/developer-guide/02-projects/otk/03-omnifest/index.md new file mode 100644 index 000000000..da88b3565 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/03-omnifest/index.md @@ -0,0 +1,42 @@ +# Omnifest + +An omnifest is the name for the YAML-based format that is used by `otk` as its input. `otk` transforms omnifests into inputs for other image build tools. To do so it works with [directives](./01-directive.md). + +## Entrypoint + +As omnifests can include other omnifests it is important to note that `otk` treats the entrypoint omnifest differently from included omnifests. The entrypoint omnifest is the file that is passed to `otk compile [file]`. This entrypoint is required to have: + +1. An [otk.version](./01-directive.md#otkversion) directive. +1. An [otk.target](./01-directive.md#otktarget) directive. + +A minimal entrypoint would look like: + +```yaml +otk.version: "1" + +otk.target.osbuild: + example: "data" +``` + +## Targets + +The [otk.target.\(.\)](./01-directive.md#otktargetconsumername) directives in the omnifest provide the umbrella to put exports under. They are namespaced to a specific consumer (e.g. `osbuild`). + +An omnifest that contains a single target will use that target by default: + +``` +€ otk compile example.yaml # example from above +# ... +``` + +When a file contains multiple targets an error will be shown; you'll have to select the target you want to export. + + +``` +€ otk compile example.yaml # contains `osbuild.ami` and `osbuild.iso` targets +CRITICAL:otk.command:INPUT contains multiple targets, `-t` is required +€ otk compile example.yaml -t osbuild.iso +# ... +``` + +Only a single target can be selected for export at a time. diff --git a/docs/developer-guide/02-projects/otk/04-best-practices.md b/docs/developer-guide/02-projects/otk/04-best-practices.md new file mode 100644 index 000000000..03fe1e43f --- /dev/null +++ b/docs/developer-guide/02-projects/otk/04-best-practices.md @@ -0,0 +1,11 @@ +# Best Practices + +As [omnifests](./03-omnifest/index.md) can become quite large here are some best practices to organise. We are trying to provide examples in our [repository](https://github.com/osbuild/otk) that adhere to these. + +## Common + +Some best practices that apply to [omnifest](./03-omnifest/index.md) files in general. + +### Definitions in the entrypoint + +Keep variable definitions in the [entrypoint](./03-omnifest/index.md#entrypoint) as much as possible. This gives a single unified place to read which variables affect the build. diff --git a/docs/developer-guide/02-projects/otk/05-deprecation.md b/docs/developer-guide/02-projects/otk/05-deprecation.md new file mode 100644 index 000000000..9e7c52968 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/05-deprecation.md @@ -0,0 +1,10 @@ +# Deprecation Policy + +`otk` has little in the way of deprecations. The important part is if or when [directives](./03-omnifest/01-directive.md) get deprecated. This will never happen with a [version](./03-omnifest/01-directive.md#otkversion). If in the future `otk` decides to deprecate an [omnifest](./03-omnifest/index.md) version then we will: + +0. Document the intent to deprecate. +1. Emit warning level logs (shown by default) for at least 6 months. +2. Disable the omnifest version by default for at least 6 months, with a flag to re-enable it. +3. Remove the omnifest version. + +This gives users a one year migration path. diff --git a/docs/developer-guide/02-projects/otk/06-security.md b/docs/developer-guide/02-projects/otk/06-security.md new file mode 100644 index 000000000..cb5e1dc1a --- /dev/null +++ b/docs/developer-guide/02-projects/otk/06-security.md @@ -0,0 +1,3 @@ +# Security + +`otk` assumes that any omnifest it processes comes from a trusted source. You should **NOT** run `otk` on an untrusted omnifest. The outputs produced by `otk` inherit the same trust level meaning that a malicious omnifest will produce a malicious output that should **NOT** be propagated to other programs. diff --git a/docs/developer-guide/02-projects/otk/07-integration.md b/docs/developer-guide/02-projects/otk/07-integration.md new file mode 100644 index 000000000..e2320f0f3 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/07-integration.md @@ -0,0 +1,3 @@ +# Integration + +`otk` will, in the future, integrate with various build systems. diff --git a/docs/developer-guide/02-projects/otk/index.md b/docs/developer-guide/02-projects/otk/index.md new file mode 100644 index 000000000..c35eb8992 --- /dev/null +++ b/docs/developer-guide/02-projects/otk/index.md @@ -0,0 +1,5 @@ +# otk + +`otk` is the omnifest toolkit. A program to transform [omnifests](./03-omnifest/index.md) to output formats such as `osbuild` manifests. + +**`otk` is currently in a proof of concept state. We are exploring a workflow and where it fits into our stack.** diff --git a/scripts/pull_otk.py b/scripts/pull_otk.py new file mode 100755 index 000000000..21d182813 --- /dev/null +++ b/scripts/pull_otk.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import pathlib +import subprocess +import sys +import shutil +import tempfile + + +def main(): + root = pathlib.Path(__file__).parent.parent + + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) + + repo = path / "repo" + repo.mkdir() + + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "https://github.com/osbuild/otk", + str(repo), + ] + ) + + dest = root / "docs/developer-guide/02-projects/otk" + + if dest.exists(): + shutil.rmtree(dest) + + shutil.copytree( + repo / "doc", + dest, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())