From 939dea17336ec1506cd8a9c18662f4c9c0b30499 Mon Sep 17 00:00:00 2001 From: Appu Goundan Date: Fri, 4 Oct 2024 13:06:47 -0400 Subject: [PATCH] Try adding tuf conformance Signed-off-by: Appu Goundan --- .github/workflows/tuf-conformance.yml | 40 ++++++++++ settings.gradle.kts | 4 +- .../main/java/dev/sigstore/tuf/Updater.java | 79 +++++++++++------- tuf-cli/README.md | 11 +++ tuf-cli/build.gradle.kts | 30 +++++++ .../java/dev/sigstore/tuf/cli/Download.java | 80 +++++++++++++++++++ .../main/java/dev/sigstore/tuf/cli/Init.java | 53 ++++++++++++ .../java/dev/sigstore/tuf/cli/Refresh.java | 60 ++++++++++++++ .../main/java/dev/sigstore/tuf/cli/Tuf.java | 36 +++++++++ 9 files changed, 364 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/tuf-conformance.yml create mode 100644 tuf-cli/README.md create mode 100644 tuf-cli/build.gradle.kts create mode 100644 tuf-cli/src/main/java/dev/sigstore/tuf/cli/Download.java create mode 100644 tuf-cli/src/main/java/dev/sigstore/tuf/cli/Init.java create mode 100644 tuf-cli/src/main/java/dev/sigstore/tuf/cli/Refresh.java create mode 100644 tuf-cli/src/main/java/dev/sigstore/tuf/cli/Tuf.java diff --git a/.github/workflows/tuf-conformance.yml b/.github/workflows/tuf-conformance.yml new file mode 100644 index 00000000..17378a20 --- /dev/null +++ b/.github/workflows/tuf-conformance.yml @@ -0,0 +1,40 @@ +name: Conformance Tests + +on: + push: + branches: + - '**' + pull_request: + workflow_dispatch: + # TODO: add cron + +jobs: + conformance: + strategy: + max-parallel: 1 + matrix: + java-version: [11, 17] + fail-fast: false + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@d156388eb19639ec20ade50009f3d199ce1e2808 # v4.1.0 + + - name: Build tuf cli + run: ./gradlew :tuf-cli:build + + - name: Unpack tuf distribution + run: tar -xvf ${{ github.workspace }}/tuf-cli/build/distributions/tuf-cli-*.tar --strip-components 1 + + - uses: theupdateframework/tuf-conformance@v2 + with: + entrypoint: ${{ github.workspace }}/bin/tuf-cli diff --git a/settings.gradle.kts b/settings.gradle.kts index 628f2228..48d85287 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -7,7 +7,9 @@ include("sigstore-java") include("sigstore-gradle:sigstore-gradle-sign-base-plugin") include("sigstore-gradle:sigstore-gradle-sign-plugin") include("sigstore-testkit") -include("sigstore-cli") include("sigstore-maven-plugin") +include("sigstore-cli") +include("tuf-cli") + include("fuzzing") diff --git a/sigstore-java/src/main/java/dev/sigstore/tuf/Updater.java b/sigstore-java/src/main/java/dev/sigstore/tuf/Updater.java index f266dbd1..789dc7de 100644 --- a/sigstore-java/src/main/java/dev/sigstore/tuf/Updater.java +++ b/sigstore-java/src/main/java/dev/sigstore/tuf/Updater.java @@ -22,6 +22,7 @@ import dev.sigstore.encryption.Keys; import dev.sigstore.encryption.signers.Verifiers; import dev.sigstore.tuf.model.*; +import dev.sigstore.tuf.model.TargetMeta.TargetData; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; @@ -69,6 +70,7 @@ public class Updater { // Mutable State private ZonedDateTime updateStartTime; + private boolean metaUpdated; Updater( Clock clock, @@ -97,19 +99,36 @@ public void update() downloadTargets(trustedMetaStore.getTargets()); } - void updateMeta() throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { - updateRoot(); - var oldTimestamp = trustedMetaStore.findTimestamp(); - updateTimestamp(); - if (Objects.equals(oldTimestamp.orElse(null), trustedMetaStore.getTimestamp()) - && trustedMetaStore.findSnapshot().isPresent() - && trustedMetaStore.findTargets().isPresent()) { - return; - } - // if we need to update or we can't find targets/timestamps locally then grab new snapshot and - // targets from remote - updateSnapshot(); - updateTargets(); + public void updateMeta() throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + try { + updateRoot(); + var oldTimestamp = trustedMetaStore.findTimestamp(); + updateTimestamp(); + if (Objects.equals(oldTimestamp.orElse(null), trustedMetaStore.getTimestamp()) + && trustedMetaStore.findSnapshot().isPresent() + && trustedMetaStore.findTargets().isPresent()) { + return; + } + // if we need to update or we can't find targets/timestamps locally then grab new snapshot and + // targets from remote + updateSnapshot(); + updateTargets(); + } finally { + metaUpdated = true; + } + } + + public void downloadTarget(String targetName) + throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + // update should run at least once before we try to grab targets + if (!metaUpdated) { + updateMeta(); + } + var targetData = trustedMetaStore.getTargets().getSignedMeta().getTargets().get(targetName); + if (targetData == null) { + throw new TargetMetadataMissingException(targetName); + } + downloadTarget(targetName, targetData); } // https://theupdateframework.github.io/specification/latest/#detailed-client-workflow @@ -459,24 +478,28 @@ void downloadTargets(Targets targets) throw new TargetMetadataMissingException(targetName); } TargetMeta.TargetData targetData = entry.getValue(); - // 9) Download target up to length specified in bytes. verify against hash. - String versionedTargetName; - if (targetData.getHashes().getSha512() != null) { - versionedTargetName = targetData.getHashes().getSha512() + "." + targetName; - } else { - versionedTargetName = targetData.getHashes().getSha256() + "." + targetName; - } + downloadTarget(targetName, targetData); + } + } - var targetBytes = targetFetcher.fetchResource(versionedTargetName, targetData.getLength()); - if (targetBytes == null) { - throw new FileNotFoundException(targetName, targetFetcher.getSource()); - } - verifyHashes(entry.getKey(), targetBytes, targetData.getHashes()); + void downloadTarget(String targetName, TargetData targetData) throws IOException { + // 9) Download target up to length specified in bytes. verify against hash. + String versionedTargetName; + if (targetData.getHashes().getSha512() != null) { + versionedTargetName = targetData.getHashes().getSha512() + "." + targetName; + } else { + versionedTargetName = targetData.getHashes().getSha256() + "." + targetName; + } - // when persisting targets use the targetname without sha512 prefix - // https://theupdateframework.github.io/specification/latest/index.html#fetch-target - targetStore.writeTarget(targetName, targetBytes); + var targetBytes = targetFetcher.fetchResource(versionedTargetName, targetData.getLength()); + if (targetBytes == null) { + throw new FileNotFoundException(targetName, targetFetcher.getSource()); } + verifyHashes(targetName, targetBytes, targetData.getHashes()); + + // when persisting targets use the targetname without sha512 prefix + // https://theupdateframework.github.io/specification/latest/index.html#fetch-target + targetStore.writeTarget(targetName, targetBytes); } @VisibleForTesting diff --git a/tuf-cli/README.md b/tuf-cli/README.md new file mode 100644 index 00000000..971907e4 --- /dev/null +++ b/tuf-cli/README.md @@ -0,0 +1,11 @@ +# Sigstore-Java Tuf CLI + +Used for conformance testing and internal processes. This is not meant for public consumption, we will not support +any usecase that uses this. + +## Usage + +### Help +``` +./gradlew tuf-cli:run +``` diff --git a/tuf-cli/build.gradle.kts b/tuf-cli/build.gradle.kts new file mode 100644 index 00000000..5363458e --- /dev/null +++ b/tuf-cli/build.gradle.kts @@ -0,0 +1,30 @@ +plugins { + id("build-logic.java") + id("application") +} + +repositories { + mavenCentral() +} + +dependencies { + implementation(project(":sigstore-java")) + implementation("info.picocli:picocli:4.7.6") + implementation("com.google.guava:guava:33.3.1-jre") + + implementation(platform("com.google.oauth-client:google-oauth-client-bom:1.36.0")) + implementation("com.google.oauth-client:google-oauth-client") + + annotationProcessor("info.picocli:picocli-codegen:4.7.6") +} + +tasks.compileJava { + options.compilerArgs.add("-Aproject=${project.group}/${project.name}") +} + +application { + mainClass.set("dev.sigstore.tuf.cli.Tuf") +} +tasks.run.configure { + workingDir = rootProject.projectDir +} diff --git a/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Download.java b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Download.java new file mode 100644 index 00000000..9de69209 --- /dev/null +++ b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Download.java @@ -0,0 +1,80 @@ +/* + * Copyright 2024 The Sigstore Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.sigstore.tuf.cli; + +import dev.sigstore.tuf.FileSystemTufStore; +import dev.sigstore.tuf.HttpFetcher; +import dev.sigstore.tuf.MetaFetcher; +import dev.sigstore.tuf.PassthroughCacheMetaStore; +import dev.sigstore.tuf.RootProvider; +import dev.sigstore.tuf.TrustedMetaStore; +import dev.sigstore.tuf.Updater; +import java.net.URL; +import java.nio.file.Path; +import java.util.concurrent.Callable; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "download", description = "download targets from a remote location") +public class Download implements Callable { + + @Option( + names = {"--metadata-dir"}, + required = true, + paramLabel = "") + Path metadataDir; + + @Option( + names = {"--metadata-url"}, + required = true, + paramLabel = "") + URL metadataUrl; + + @Option( + names = {"--target-name"}, + required = true, + paramLabel = "") + String targetName; + + @Option( + names = {"--target-base-url"}, + required = true, + paramLabel = "") + URL targetBaseUrl; + + @Option( + names = {"--target-dir"}, + required = true, + paramLabel = "") + Path targetDir; + + @Override + public Integer call() throws Exception { + var fsStore = FileSystemTufStore.newFileSystemStore(metadataDir); + var tuf = + Updater.builder() + .setTrustedMetaStore( + TrustedMetaStore.newTrustedMetaStore( + PassthroughCacheMetaStore.newPassthroughMetaCache(fsStore))) + .setTrustedRootPath(RootProvider.fromFile(metadataDir.resolve("root.json"))) + .setMetaFetcher(MetaFetcher.newFetcher(HttpFetcher.newFetcher(metadataUrl))) + .setTargetFetcher(HttpFetcher.newFetcher(targetBaseUrl)) + .setTargetStore(fsStore) + .build(); + tuf.downloadTarget(targetName); + return 0; + } +} diff --git a/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Init.java b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Init.java new file mode 100644 index 00000000..4d34bff6 --- /dev/null +++ b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Init.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 The Sigstore Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.sigstore.tuf.cli; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Callable; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "init", description = "initialize a local tuf repo") +public class Init implements Callable { + + @Parameters(arity = "1", paramLabel = "") + Path trustedRoot; + + @Option( + names = {"--metadata-dir"}, + required = true, + paramLabel = "") + Path metadataDir; + + @Override + public Integer call() throws Exception { + if (!Files.isRegularFile(trustedRoot)) { + throw new IllegalArgumentException(trustedRoot + " is not a regular file"); + } + if (Files.exists(metadataDir)) { + if (!Files.isDirectory(metadataDir)) { + throw new IllegalArgumentException(metadataDir + " is not a directory"); + } + } else { + Files.createDirectories(metadataDir); + } + + Files.copy(trustedRoot, metadataDir.resolve("root.json")); + return 0; + } +} diff --git a/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Refresh.java b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Refresh.java new file mode 100644 index 00000000..2f27a0b8 --- /dev/null +++ b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Refresh.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024 The Sigstore Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.sigstore.tuf.cli; + +import dev.sigstore.tuf.FileSystemTufStore; +import dev.sigstore.tuf.HttpFetcher; +import dev.sigstore.tuf.MetaFetcher; +import dev.sigstore.tuf.PassthroughCacheMetaStore; +import dev.sigstore.tuf.RootProvider; +import dev.sigstore.tuf.TrustedMetaStore; +import dev.sigstore.tuf.Updater; +import java.net.URL; +import java.nio.file.Path; +import java.util.concurrent.Callable; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "refresh", description = "update local tuf metadata from the repository") +public class Refresh implements Callable { + + @Option( + names = {"--metadata-dir"}, + required = true, + paramLabel = "") + Path metadataDir; + + @Option( + names = {"--metadata-url"}, + required = true, + paramLabel = "") + URL metadataUrl; + + @Override + public Integer call() throws Exception { + var fsStore = FileSystemTufStore.newFileSystemStore(metadataDir); + var tuf = + Updater.builder() + .setTrustedMetaStore( + TrustedMetaStore.newTrustedMetaStore( + PassthroughCacheMetaStore.newPassthroughMetaCache(fsStore))) + .setTrustedRootPath(RootProvider.fromFile(metadataDir.resolve("root.json"))) + .setMetaFetcher(MetaFetcher.newFetcher(HttpFetcher.newFetcher(metadataUrl))) + .build(); + tuf.update(); + return 0; + } +} diff --git a/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Tuf.java b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Tuf.java new file mode 100644 index 00000000..0a7cab12 --- /dev/null +++ b/tuf-cli/src/main/java/dev/sigstore/tuf/cli/Tuf.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 The Sigstore Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.sigstore.tuf.cli; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +@Command( + name = "tuf", + mixinStandardHelpOptions = true, + + subcommands = {Init.class, Refresh.class, Download.class}) +public class Tuf { + @Spec CommandSpec spec; + + public static void main(String[] args) { + System.out.println(String.join(" ", args)); + int exitCode = new CommandLine(new Tuf()).execute(args); + System.exit(exitCode); + } +}