diff --git a/.codespellrc b/.codespellrc index cb01ec544..afe8ce0a9 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,4 +1,4 @@ [codespell] # skipping auto generated folders skip = ./.tox,./.mypy_cache,./target,*/LICENSE,./venv,*/sql_dialect_keywords.json -ignore-words-list = afterall,assertIn \ No newline at end of file +ignore-words-list = afterall,assertIn, crate \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e8aa35db1..b25422655 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ # The packages are installed in the `/autoinstrumentation` directory. This is required as when instrumenting the pod by CWOperator, # one init container will be created to copy all the content in `/autoinstrumentation` directory to app's container. Then # update the `PYTHONPATH` environment variable accordingly. Then in the second stage, copy the directory to `/autoinstrumentation`. + +# Stage 1: Install ADOT Python in the /operator-build folder FROM python:3.11 AS build WORKDIR /operator-build @@ -18,11 +20,42 @@ RUN sed -i "/opentelemetry-exporter-otlp-proto-grpc/d" ./aws-opentelemetry-distr RUN mkdir workspace && pip install --target workspace ./aws-opentelemetry-distro -FROM public.ecr.aws/amazonlinux/amazonlinux:minimal +# Stage 2: Build the cp-utility binary +FROM rust:1.75 as builder + +WORKDIR /usr/src/cp-utility +COPY ./tools/cp-utility . + +## TARGETARCH is defined by buildx +# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETARCH + +# Run validations and audit only on amd64 because it is faster and those two steps +# are only used to validate the source code and don't require anything that is +# architecture specific. + +# Validations +# Validate formatting +RUN if [ $TARGETARCH = "amd64" ]; then rustup component add rustfmt && cargo fmt --check ; fi + +# Audit dependencies +RUN if [ $TARGETARCH = "amd64" ]; then cargo install cargo-audit && cargo audit ; fi + + +# Cross-compile based on the target platform. +RUN if [ $TARGETARCH = "amd64" ]; then export ARCH="x86_64" ; \ + elif [ $TARGETARCH = "arm64" ]; then export ARCH="aarch64" ; \ + else false; \ + fi \ + && rustup target add ${ARCH}-unknown-linux-musl \ + && cargo test --target ${ARCH}-unknown-linux-musl \ + && cargo install --target ${ARCH}-unknown-linux-musl --path . --root . + +# Stage 3: Build the distribution image by copying the THIRD-PARTY-LICENSES, the custom built cp command from stage 2, and the installed ADOT Python from stage 1 to their respective destinations +FROM scratch # Required to copy attribute files to distributed docker images ADD THIRD-PARTY-LICENSES ./THIRD-PARTY-LICENSES +COPY --from=builder /usr/src/cp-utility/bin/cp-utility /bin/cp COPY --from=build /operator-build/workspace /autoinstrumentation - -RUN chmod -R go+r /autoinstrumentation diff --git a/tools/cp-utility/.cargo/config.toml b/tools/cp-utility/.cargo/config.toml new file mode 100644 index 000000000..5f92a1de4 --- /dev/null +++ b/tools/cp-utility/.cargo/config.toml @@ -0,0 +1,3 @@ +# Use git CLI to fetch the source code instead of relying on the rust git implementation +[net] +git-fetch-with-cli = true diff --git a/tools/cp-utility/Cargo.lock b/tools/cp-utility/Cargo.lock new file mode 100644 index 000000000..a1aa5dd68 --- /dev/null +++ b/tools/cp-utility/Cargo.lock @@ -0,0 +1,221 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cp-utility" +version = "0.1.0" +dependencies = [ + "tempfile", + "uuid", +] + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rustix" +version = "0.38.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "uuid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" +dependencies = [ + "getrandom", + "rand", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" diff --git a/tools/cp-utility/Cargo.toml b/tools/cp-utility/Cargo.toml new file mode 100644 index 000000000..7169660e4 --- /dev/null +++ b/tools/cp-utility/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "cp-utility" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# No dependencies here + +[dev-dependencies] +# dependencies only used during tests +tempfile = "3.9.0" +uuid = { version = "1.5.0", features = ["v4", "fast-rng"] } + +[profile.release] +# Levers to optimize the binary for size +strip = true # Strip symbols +opt-level = "z" # Size optimization +lto = true # linking time optimizations + + diff --git a/tools/cp-utility/README.md b/tools/cp-utility/README.md new file mode 100644 index 000000000..f67227a29 --- /dev/null +++ b/tools/cp-utility/README.md @@ -0,0 +1,57 @@ +# Introduction + +This copy utility is intended to be used as a base image for OpenTelemetry Operator +autoinstrumentation images. The copy utility will allow the ADOT Java agent jar to be +copied from the init container to the final destination volume. + +## Development + +### Pre-requirements +* Install rust + +``` +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +* Install rustfmt + +``` +rustup component add rustfmt +``` + +### Development + +* Auto formatting the code + +This step is important and it might fail the build if the files are not properly +formatted. + +``` +cargo fmt +``` + +* Testing the code +``` +cargo test +``` + +* Building the code + +``` +cargo build +``` + +NOTE: this will build the code for tests locally. It will not statically link the libc used by it. + + +* Building the code statically linked + +``` +cargo build --target x86_64-unknown-linux-musl +``` + + +### Docker image + +In the root of this project, there is a Dockerfile that is supposed to be used during release. +This Dockerfile can be used with buildx to generate images for the arm64 and x86_64 platforms. diff --git a/tools/cp-utility/src/main.rs b/tools/cp-utility/src/main.rs new file mode 100644 index 000000000..6e140611b --- /dev/null +++ b/tools/cp-utility/src/main.rs @@ -0,0 +1,345 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +use std::collections::VecDeque; +use std::env; +use std::fs; +use std::io; +use std::os::unix; +use std::path::Path; +use std::path::PathBuf; +use std::process; + +/// A type of copy operation +#[derive(Debug, PartialEq)] +enum CopyType { + /// equivalent to cp + SingleFile, + /// equivalent to cp -a + Archive, +} + +/// Encapsulate a copy operation +struct CopyOperation { + /// The source path + source: PathBuf, + /// The destination path + destination: PathBuf, + /// The type of copy being performed + copy_type: CopyType, +} + +/// Parse command line arguments and transform into `CopyOperation` +fn parse_args(args: Vec<&str>) -> io::Result { + if !(args.len() == 3 || args.len() == 4 && args[1].eq("-a")) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Invalid parameters. Expected cp [-a] ", + )); + } + + if args.len() == 4 { + return Ok(CopyOperation { + source: PathBuf::from(args[2]), + destination: PathBuf::from(args[3]), + copy_type: CopyType::Archive, + }); + } + + Ok(CopyOperation { + source: PathBuf::from(args[1]), + destination: PathBuf::from(args[2]), + copy_type: CopyType::SingleFile, + }) +} + +/// Execute the copy operation +fn do_copy(operation: CopyOperation) -> io::Result<()> { + match operation.copy_type { + CopyType::Archive => copy_archive(&operation.source, &operation.destination)?, + CopyType::SingleFile => fs::copy(&operation.source, &operation.destination).map(|_| ())?, + }; + Ok(()) +} + +// Execute the recursive type of copy operation +fn copy_archive(source: &Path, dest: &Path) -> io::Result<()> { + let mut stack = VecDeque::new(); + stack.push_back((source.to_path_buf(), dest.to_path_buf())); + while let Some((current_source, current_dest)) = stack.pop_back() { + if current_source.is_symlink() { + let target = current_source.read_link()?; + unix::fs::symlink(target, ¤t_dest)?; + } else if current_source.is_dir() { + if !current_dest.exists() { + fs::create_dir(¤t_dest)?; + } + for entry in fs::read_dir(current_source)? { + let next_source = entry?.path(); + let next_dest = + current_dest + .clone() + .join(next_source.file_name().ok_or(io::Error::new( + io::ErrorKind::InvalidInput, + "Invalid source file", + ))?); + stack.push_back((next_source, next_dest)); + } + } else if current_source.is_file() { + fs::copy(current_source, current_dest)?; + } + } + + Ok(()) +} + +fn main() { + let original_args: Vec = env::args().collect(); + let args = original_args.iter().map(|x| x.as_str()).collect(); + + let operation = parse_args(args).unwrap_or_else(|err| { + eprintln!("Error parsing arguments: {err}"); + process::exit(1); + }); + + do_copy(operation).unwrap_or_else(|err| { + eprintln!("Error copying files: {err}"); + process::exit(2); + }); +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + io::Write, + os::unix, + path::{Path, PathBuf}, + }; + + use crate::{do_copy, parse_args, CopyOperation, CopyType}; + use uuid; + + #[test] + fn test_parser_archive() { + // prepare + let input = vec!["cp", "-a", "foo.txt", "dest.txt"]; + + // act + let result = parse_args(input).unwrap(); + + // assert + assert_eq!(result.source, PathBuf::from("foo.txt")); + assert_eq!(result.destination, PathBuf::from("dest.txt")); + assert_eq!(result.copy_type, CopyType::Archive) + } + + #[test] + fn test_parser_single() { + // prepare + let input: Vec<&str> = vec!["cp", "foo.txt", "dest.txt"]; + + // act + let result = parse_args(input).unwrap(); + + // assert + assert_eq!(result.source, PathBuf::from("foo.txt")); + assert_eq!(result.destination, PathBuf::from("dest.txt")); + assert_eq!(result.copy_type, CopyType::SingleFile) + } + + #[test] + fn parser_failure() { + // prepare + let inputs = vec![ + vec!["cp", "-r", "foo.txt", "bar.txt"], + vec!["cp", "-a", "param1", "param2", "param3"], + vec!["cp", "param1", "param2", "param3"], + ]; + + for input in inputs.into_iter() { + // act + let result = parse_args(input.clone()); + + // assert + assert!(result.is_err(), "input should fail {:?}", input); + } + } + + #[test] + fn test_copy_single() { + // prepare + let tempdir = tempfile::tempdir().unwrap(); + let test_base = tempdir.path().to_path_buf(); + + create_file(&test_base, "foo.txt"); + + let source = test_base.join("foo.txt"); + let dest = test_base.join("bar.txt"); + let single_copy = CopyOperation { + copy_type: CopyType::SingleFile, + source: source.clone(), + destination: dest.clone(), + }; + + // act + do_copy(single_copy).unwrap(); + + // assert + assert_same_file(&source, &dest) + } + + #[test] + fn single_cannot_copy_directory() { + // prepare + let tempdir = tempfile::tempdir().unwrap(); + let test_base = tempdir.path().to_path_buf(); + + create_dir(&test_base, "somedir"); + + // act + let single_copy = CopyOperation { + copy_type: CopyType::SingleFile, + source: test_base.join("somedir"), + destination: test_base.join("somewhereelse"), + }; + let result = do_copy(single_copy); + + // assert + assert!(result.is_err()); + } + + #[test] + fn test_copy_archive() { + // prepare + let tempdir = tempfile::tempdir().unwrap(); + let test_base = tempdir.path().to_path_buf(); + ["foo", "foo/foo0", "foo/foo1", "foo/bar"] + .iter() + .for_each(|x| create_dir(&test_base, x)); + let files = [ + "foo/file1.txt", + "foo/file2.txt", + "foo/foo1/file3.txt", + "foo/bar/file4.txt", + ]; + files.iter().for_each(|x| create_file(&test_base, x)); + [("foo/symlink1.txt", "./file1.txt")] + .iter() + .for_each(|(x, y)| create_symlink(&test_base, x, y)); + + // act + let recursive_copy = CopyOperation { + copy_type: CopyType::Archive, + source: test_base.join("foo"), + destination: test_base.join("bar"), + }; + do_copy(recursive_copy).unwrap(); + + // assert + files.iter().for_each(|x| { + assert_same_file( + &test_base.join(x), + &test_base.join(x.replace("foo/", "bar/")), + ) + }); + assert_same_file( + &test_base.join("foo/symlink1.txt"), + &test_base.join("bar/symlink1.txt"), + ); + + assert_same_link( + &test_base.join("foo/symlink1.txt"), + &test_base.join("bar/symlink1.txt"), + ) + } + + #[test] + fn test_copy_archive_destination_exists() { + // prepare + let tempdir = tempfile::tempdir().unwrap(); + let test_base = tempdir.path().to_path_buf(); + ["foo", "foo/foo0", "foo/foo1", "foo/bar"] + .iter() + .for_each(|x| create_dir(&test_base, x)); + let files = [ + "foo/file1.txt", + "foo/file2.txt", + "foo/foo1/file3.txt", + "foo/bar/file4.txt", + ]; + files.iter().for_each(|x| create_file(&test_base, x)); + [("foo/symlink1.txt", "./file1.txt")] + .iter() + .for_each(|(x, y)| create_symlink(&test_base, x, y)); + create_dir(&test_base, "bar"); + + // act + let recursive_copy = CopyOperation { + copy_type: CopyType::Archive, + source: test_base.join("foo"), + destination: test_base.join("bar"), + }; + do_copy(recursive_copy).unwrap(); + + // assert + files.iter().for_each(|x| { + assert_same_file( + &test_base.join(x), + &test_base.join(x.replace("foo/", "bar/")), + ) + }); + + assert_same_link( + &test_base.join("foo/symlink1.txt"), + &test_base.join("bar/symlink1.txt"), + ) + } + + // Utility functions used in the tests + fn create_dir(base: &Path, dir: &str) { + fs::create_dir_all(base.to_path_buf().join(dir)).unwrap(); + } + + fn create_file(base: &Path, file: &str) { + let mut file = fs::File::create(base.to_path_buf().join(file)).unwrap(); + file.write_fmt(format_args!("{}", uuid::Uuid::new_v4().to_string())) + .unwrap(); + } + + fn create_symlink(base: &Path, file: &str, target: &str) { + unix::fs::symlink(Path::new(target), &base.to_path_buf().join(file)).unwrap(); + } + + fn assert_same_file(source: &Path, dest: &Path) { + assert!(source.exists()); + assert!(dest.exists()); + assert!(source.is_file()); + assert!(dest.is_file()); + + assert_eq!( + fs::read_to_string(source).unwrap(), + fs::read_to_string(dest).unwrap() + ); + } + + fn assert_same_link(source: &Path, dest: &Path) { + assert!(source.exists()); + assert!(dest.exists()); + assert!(source.is_symlink()); + assert!(dest.is_symlink()); + + assert_eq!(fs::read_link(source).unwrap(), fs::read_link(dest).unwrap()); + } +}