Skip to content

Commit

Permalink
use local container registry on persistent volume rather then tar files
Browse files Browse the repository at this point in the history
  • Loading branch information
@itewk
itewk committed Aug 4, 2021
1 parent 1a93acc commit e0c8bac
Show file tree
Hide file tree
Showing 15 changed files with 1,514 additions and 929 deletions.
4 changes: 2 additions & 2 deletions src/ploigos_step_runner/results/workflow_result.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ def get_artifact_value(
): # pylint: disable=too-many-boolean-expressions
"""Search for an artifact.
If step_name, sub_step_name, or environment are provided ensure the artifact comes
from the first
If step_name, sub_step_name, or environment are not provided ensure the artifact comes
from the last step that returned that artifact.
1. if step_name is provided, look for the artifact in the step
2. elif step_name and sub_step_name is provided, look for the artifact in the step/sub_step
Expand Down
17 changes: 11 additions & 6 deletions src/ploigos_step_runner/step_implementers/container_image_static_compliance_scan/openscap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,15 @@
Configuration Key | Required? | Default | Description
-------------------------------|-----------|---------|-----------
`image-tar-file` | Yes | | Path to container image tar file to scan
`container-image-tag` | Yes | | Container image tag to scan.
`oscap-input-definitions-uri` | Yes | | URI to the OpenSCAP definitions file \
to do the evaluation with. \
Must use protocol file://|http://|https://.
| | | Must have file extension .xml|.bz2.
`oscap-profile` | Yes | | OpenSCAP profile to evaluate.
Must use protocol file://|http://|https://. \
Must have file extension .xml|.bz2.
`oscap-profile` | No | | OpenSCAP profile to evaluate.
`oscap-tailoring-uri` | No | | URI to OpenSCAP tailoring file \
to do the evaluation with. \
Must use protocol \
file://|http://|https://. \
Must use protocol file://|http://|https://. \
Must have file extension .xml|.bz2.
`oscap-fetch-remote-resources` | No | True | For Source DataStream and XCCDF files \
that have remote references fetch them if \
Expand All @@ -34,6 +33,12 @@
remote resources and this is not True. \
For disconnected environments the remote \
internal mirror.
`[container-image-pull-repository-type, container-image-repository-type]` \
| Yes | 'containers-storage:' \
| \
Container repository type for the pull image source. \
See https://github.com/containers/skopeo for valid \
options.
Result Artifacts
----------------
Expand Down
17 changes: 11 additions & 6 deletions ...oigos_step_runner/step_implementers/container_image_static_vulnerability_scan/openscap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,16 @@
* previous step results
Configuration Key | Required? | Default | Description
-------------------------------|-----------|---------|------------
`image-tar-file` | Yes | | Path to container image tar file to scan
-------------------------------|-----------|---------|-----------
`container-image-tag` | Yes | | Container image tag to scan.
`oscap-input-definitions-uri` | Yes | | URI to the OpenSCAP definitions file \
to do the evaluation with. \
Must use protocol file://|http://|https://.
| | | Must have file extension .xml|.bz2.
Must use protocol file://|http://|https://. \
Must have file extension .xml|.bz2.
`oscap-profile` | No | | OpenSCAP profile to evaluate.
`oscap-tailoring-uri` | No | | URI to OpenSCAP tailoring file \
to do the evaluation with. \
Must use protocol \
file://|http://|https://. \
Must use protocol file://|http://|https://. \
Must have file extension .xml|.bz2.
`oscap-fetch-remote-resources` | No | True | For Source DataStream and XCCDF files \
that have remote references fetch them if \
Expand All @@ -34,6 +33,12 @@
remote resources and this is not True. \
For disconnected environments the remote \
internal mirror.
`[container-image-pull-repository-type, container-image-repository-type]` \
| Yes | 'containers-storage:' \
| \
Container repository type for the pull image source. \
See https://github.com/containers/skopeo for valid \
options.
Result Artifacts
----------------
Expand Down
202 changes: 128 additions & 74 deletions src/ploigos_step_runner/step_implementers/create_container_image/buildah.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,41 +9,64 @@
* runtime configuration
* previous step results
Configuration Key | Required? | Default | Description
------------------|-----------|---------|-----------
`imagespecfile` | True | `'Containerfile'` \
| File defining the container image
`context` | True | `'.'` | Context to build the container image in
`tls-verify` | True | `True` | Whether to verify TLS when pulling parent images
`format` | True | `'oci'` | format of the built image's manifest and metadata
Configuration Key | Required? | Default | Description
-------------------|-----------|---------|-----------
`imagespecfile` | True | `'Containerfile'` \
| File defining the container image
`context` | True | `'.'` | Context to build the container image in
`tls-verify` | True | `True` | Whether to verify TLS when pulling parent images
`format` | True | `'oci'` | format of the built image's manifest and metadata
`containers-config-auth-file` \
| True | `'~/.buildah-auth.json'` \
| Path to the container registry authentication \
file to use for container registry authentication.
| False | | Path to the container registry authentication \
file to use for container registry authentication. \
If one is not provided one will be created in the \
working directory.
`container-image-version` \
| True | | Version to use when building the container image
| True | | Version to use when building the container image
`organization` | True | | Used in built container image tag
`application_name` | True | | Used in built container image tag
`service_name` | True | | Used in built container image tag
`container-registries` \
| False | | Hash of container registries to authenticate with.
Result Artifacts
----------------
Results artifacts output by this step.
Result Artifact Key | Description
--------------------------|------------
`container-image-version` | Container version to tag built image with
`image-tar-file` | Path to the built container image as a tar file
"""
Result Artifact Key | Description
-------------------------------|------------
`container-image-registry-uri` | Registry URI poriton of the container image tag \
of the built container image.
`container-image-registry-organization` \
| Organization portion of the container image tag \
of the built container image.
`container-image-repository` | Repository portion of the container image tag \
of the built container image.
`container-image-name` | Another way to reference the \
repository portion of the container image tag \
of the built container image.
`container-image-version` | Version portion of the container image tag \
of the built container image.
`container-image-tag` | Full container image tag of the built container, \
including the registry URI. <br/> \
Takes the form of: \
`container-image-registry-organization/container-image-repository:container-image-version`
`container-image-short-tag` | Short container image tag of the built container image, \
excluding the registry URI. <br/> \
Takes the form of: \
`container-image-registry-uri/container-image-registry-organization/container-image-repository:container-image-version`
""" # pylint: disable=line-too-long

import os
import sys
from pathlib import Path
from distutils import util

import sh
from ploigos_step_runner import StepImplementer, StepResult
from ploigos_step_runner.utils.containers import container_registries_login

DEFAULT_CONFIG = {
# Path to the container registry authentication file to read and write to/from.
'containers-config-auth-file': os.path.join(Path.home(), '.buildah-auth.json'),

# Image specification file name
'imagespecfile': 'Containerfile',

Expand All @@ -58,11 +81,11 @@
}

REQUIRED_CONFIG_OR_PREVIOUS_STEP_RESULT_ARTIFACT_KEYS = [
'containers-config-auth-file',
'imagespecfile',
'context',
'tls-verify',
'format',
'organization',
'service-name',
'application-name'
]
Expand Down Expand Up @@ -103,6 +126,29 @@ def _required_config_or_result_keys():
"""
return REQUIRED_CONFIG_OR_PREVIOUS_STEP_RESULT_ARTIFACT_KEYS

def _validate_required_config_or_previous_step_result_artifact_keys(self):
"""Validates that the required configuration keys or previous step result artifacts
are set and have valid values.
Validates that:
* required configuration is given
* given 'imagespecfile' exists
Raises
------
AssertionError
If step configuration or previous step result artifacts have invalid required values
"""
super()._validate_required_config_or_previous_step_result_artifact_keys()

# if pom-file has value verify file exists
# If it doesn't have value and is required function will have already failed
image_spec_file = self.get_value('imagespecfile')
context = self.get_value('context')
image_spec_file_full_path = os.path.join(context, image_spec_file)
assert os.path.exists(image_spec_file_full_path), \
f'Given imagespecfile ({image_spec_file}) does not exist in given context ({context}).'

def _run_step(self):
"""Runs the step implemented by this StepImplementer.
Expand All @@ -113,38 +159,33 @@ def _run_step(self):
"""
step_result = StepResult.from_step_implementer(self)

context = self.get_value('context')
# get config
image_spec_file = self.get_value('imagespecfile')
image_spec_file_location = os.path.join(context, image_spec_file)
application_name = self.get_value('application-name')
service_name = self.get_value('service-name')
tls_verify = self.get_value('tls-verify')
if isinstance(tls_verify, str):
tls_verify = bool(util.strtobool(tls_verify))

if not os.path.exists(image_spec_file_location):
step_result.success = False
step_result.message = 'Image specification file does not exist in location: ' \
f'{image_spec_file_location}'
return step_result

image_tag_version = self.get_value('container-image-version')
if image_tag_version is None:
image_tag_version = 'latest'
# create local build tag
image_version = self.get_value('container-image-version')
if image_version is None:
image_version = 'latest'
print('No image tag version found in metadata. Using latest')

destination = "localhost/{application_name}/{service_name}".format(
application_name=application_name,
service_name=service_name
)
tag = "{destination}:{version}".format(
destination=destination,
version=image_tag_version
)
image_registry_uri = 'localhost'
image_registry_organization = self.get_value('organization')
image_repository = f"{self.get_value('application-name')}-{self.get_value('service-name')}"
short_tag = f"{image_registry_organization}/{image_repository}:{image_version}"
build_tag = f"{image_registry_uri}/{short_tag}"

try:
# login to any provider container registries
# NOTE: important to specify the auth file because depending on the context this is
# being run in python process may not have permissions to default location
containers_config_auth_file = self.get_value('containers-config-auth-file')
if not containers_config_auth_file:
containers_config_auth_file = os.path.join(
self.work_dir_path,
'container-auth.json'
)
container_registries_login(
registries=self.get_value('container-registries'),
containers_config_auth_file=containers_config_auth_file,
Expand All @@ -156,48 +197,61 @@ def _run_step(self):
'--format=' + self.get_value('format'),
'--tls-verify=' + str(tls_verify).lower(),
'--layers', '-f', image_spec_file,
'-t', tag,
'-t', build_tag,
'--authfile', containers_config_auth_file,
context,
self.get_value('context'),
_out=sys.stdout,
_err=sys.stderr,
_tee='err'
)

step_result.add_artifact(
name='container-image-version',
value=tag
)
except sh.ErrorReturnCode as error: # pylint: disable=undefined-variable
step_result.success = False
step_result.message = 'Issue invoking buildah bud with given image ' \
f'specification file ({image_spec_file}): {error}'
return step_result

image_tar_file = f'image-{application_name}-{service_name}-{image_tag_version}.tar'
image_tar_path = os.path.join(self.work_dir_path, image_tar_file)
try:
# Check to see if the tar docker-archive file already exists
# this needs to be run as buildah does not support overwritting
# existing files.
if os.path.exists(image_tar_path):
os.remove(image_tar_path)
sh.buildah.push( # pylint: disable=no-member
tag,
"docker-archive:" + image_tar_path,
_out=sys.stdout,
_err=sys.stderr,
_tee='err'
)

step_result.add_artifact(
name='image-tar-file',
value=image_tar_path
)
except sh.ErrorReturnCode as error: # pylint: disable=undefined-variable
step_result.success = False
step_result.message = f'Issue invoking buildah push to tar file ' \
f'({image_tar_path}): {error}'
return step_result
# add artifacts
step_result.add_artifact(
name='container-image-registry-uri',
value=image_registry_uri,
description='Registry URI poriton of the container image tag' \
' of the built container image.'
)
step_result.add_artifact(
name='container-image-registry-organization',
value=image_registry_organization,
description='Organization portion of the container image tag' \
' of the built container image.'
)
step_result.add_artifact(
name='container-image-repository',
value=image_repository,
description='Repository portion of the container image tag' \
' of the built container image.'
)
step_result.add_artifact(
name='container-image-name',
value=image_repository,
description='Another way to reference the' \
' repository portion of the container image tag of the built container image.'
)
step_result.add_artifact(
name='container-image-version',
value=image_version,
description='Version portion of the container image tag of the built container image.'
)
step_result.add_artifact(
name='container-image-tag',
value=build_tag,
description='Full container image tag of the built container,' \
' including the registry URI.'
)
step_result.add_artifact(
name='container-image-short-tag',
value=short_tag,
description='Short container image tag of the built container image,' \
' excluding the registry URI.'
)

return step_result
Loading

0 comments on commit e0c8bac

Please sign in to comment.