Merge pull request #656 from sigstore/optional_snapshots

TUF allows optional hashes and length on snapshots

sigstore-java/src/main/java/dev/sigstore/tuf/Updater.java

@@ -308,17 +308,20 @@ Snapshot updateSnapshot(Root root, Timestamp timestamp)
             Role.Name.SNAPSHOT,
             timestampSnapshotVersion,
             Snapshot.class,
-            timestamp.getSignedMeta().getSnapshotMeta().getLength());
+            timestamp.getSignedMeta().getSnapshotMeta().getLengthOrDefault());
     if (snapshotResult.isEmpty()) {
       throw new FileNotFoundException(
           timestampSnapshotVersion + ".snapshot.json", fetcher.getSource());
     }
-    // 2) check against timestamp.snapshot.hash
+    // 2) check against timestamp.snapshot.hash, this is optional, the fallback is
+    // that the version must match, which is handled in (4).
     var snapshot = snapshotResult.get();
-    verifyHashes(
-        "snapshot",
-        snapshot.getRawBytes(),
-        timestamp.getSignedMeta().getSnapshotMeta().getHashes());
+    if (timestamp.getSignedMeta().getSnapshotMeta().getHashes().isPresent()) {
+      verifyHashes(
+          "snapshot",
+          snapshot.getRawBytes(),
+          timestamp.getSignedMeta().getSnapshotMeta().getHashes().get());
+    }
     // 3) Check against threshold of root signing keys, else fail
     verifyDelegate(root, snapshot.getMetaResource());
     // 4) Check snapshot.version matches timestamp.snapshot.version, else fail.
@@ -392,17 +395,23 @@ Targets updateTargets(Root root, Snapshot snapshot)
     SnapshotMeta.SnapshotTarget targetMeta = snapshot.getSignedMeta().getTargetMeta("targets.json");
     var targetsResultMaybe =
         fetcher.getMeta(
-            Role.Name.TARGETS, targetMeta.getVersion(), Targets.class, targetMeta.getLength());
+            Role.Name.TARGETS,
+            targetMeta.getVersion(),
+            Targets.class,
+            targetMeta.getLengthOrDefault());
     if (targetsResultMaybe.isEmpty()) {
       throw new FileNotFoundException(
           targetMeta.getVersion() + ".targets.json", fetcher.getSource());
     }
     var targetsResult = targetsResultMaybe.get();
-    // 2) check hash against snapshot.targets.hash, else fail.
-    verifyHashes(
-        targetMeta.getVersion() + ".targets.json",
-        targetsResult.getRawBytes(),
-        targetMeta.getHashes());
+    // 2) check hash against snapshot.targets.hash, else just make sure versions match, handled
+    // by (4)
+    if (targetMeta.getHashes().isPresent()) {
+      verifyHashes(
+          targetMeta.getVersion() + ".targets.json",
+          targetsResult.getRawBytes(),
+          targetMeta.getHashes().get());
+    }
     // 3) check against threshold of keys as specified by trusted root.json
     verifyDelegate(root, targetsResult.getMetaResource());
     // 4) check targets.version == snapshot.targets.version, else fail.

sigstore-java/src/main/java/dev/sigstore/tuf/model/SnapshotMeta.java

@@ -16,8 +16,10 @@
 package dev.sigstore.tuf.model;
 
 import java.util.Map;
+import java.util.Optional;
 import org.immutables.gson.Gson;
 import org.immutables.value.Value;
+import org.immutables.value.Value.Derived;
 
 /**
  * The snapshot.json metadata file lists version numbers of all metadata files other than
@@ -29,6 +31,12 @@
 @Value.Immutable
 public interface SnapshotMeta extends TufMeta {
 
+  /**
+   * If no length is provided, use a default, we just use the same default as the python client:
+   * https://github.com/theupdateframework/python-tuf/blob/22080157f438357935bfcb25b5b8429f4e4f610e/tuf/ngclient/config.py#L49
+   */
+  Integer DEFAULT_MAX_LENGTH = 2000000;
+
   /** Maps role and delegation role names (e.g. "targets.json") to snapshot metadata. */
   Map<String, SnapshotTarget> getMeta();
 
@@ -40,11 +48,20 @@ default SnapshotTarget getTargetMeta(String targetName) {
   @Value.Immutable
   interface SnapshotTarget {
 
-    /** The valid hashes for the given target's metadata. */
-    Hashes getHashes();
+    /** The valid hashes for the given target's metadata. This is optional and may not be present */
+    Optional<Hashes> getHashes();
+
+    /**
+     * The length in bytes of the given target's metadata. This is optional and may not be present,
+     * use {@link #getLengthOrDefault} to delegate to the client default config.
+     */
+    Optional<Integer> getLength();
 
-    /** The length in bytes of the given target's metadata. */
-    int getLength();
+    /** The length in bytes of the given target's metadata, or a default if not present */
+    @Derived
+    default Integer getLengthOrDefault() {
+      return getLength().orElse(DEFAULT_MAX_LENGTH);
+    }
 
     /** The expected version of the given target's metadata. */
     int getVersion();

sigstore-java/src/test/java/dev/sigstore/tuf/UpdaterTest.java

@@ -74,6 +74,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -969,6 +970,31 @@ public void testVerifyDelegate_goodSigsAndKeysButNotInRole()
     }
   }
 
+  @Test
+  public void testUpdate_snapshotsAndTimestampHaveNoSizeAndNoHashesInMeta()
+      throws IOException, NoSuchAlgorithmException, InvalidKeySpecException, InvalidKeyException {
+    setupMirror(
+        "synthetic/no-size-no-hash-snapshot-timestamp",
+        "2.root.json",
+        "timestamp.json",
+        "3.snapshot.json");
+    var updater =
+        createTimeStaticUpdater(
+            localStorePath,
+            UPDATER_SYNTHETIC_TRUSTED_ROOT,
+            "2022-11-20T18:07:27Z"); // one day after
+    Root root = updater.updateRoot();
+    Timestamp timestamp = updater.updateTimestamp(root).get();
+    Snapshot snapshot = updater.updateSnapshot(root, timestamp);
+
+    Assertions.assertTrue(timestamp.getSignedMeta().getSnapshotMeta().getHashes().isEmpty());
+    Assertions.assertTrue(timestamp.getSignedMeta().getSnapshotMeta().getLength().isEmpty());
+    Assertions.assertTrue(
+        snapshot.getSignedMeta().getMeta().get("targets.json").getHashes().isEmpty());
+    Assertions.assertTrue(
+        snapshot.getSignedMeta().getMeta().get("targets.json").getLength().isEmpty());
+  }
+
   @Test
   public void canCreateMultipleUpdaters() throws IOException {
     createTimeStaticUpdater(localStorePath, UPDATER_REAL_TRUSTED_ROOT);

sigstore-java/src/test/java/dev/sigstore/tuf/model/TestTufJsonLoading.java

@@ -96,11 +96,11 @@ public void loadSnapshotJson() throws IOException {
     assertNotNull(rekorSnapshot.getHashes());
     assertEquals(
         "9d2e1a5842937d8e0d3e3759170b0ad15c56c5df36afc5cf73583ddd283a463b",
-        rekorSnapshot.getHashes().getSha256());
+        rekorSnapshot.getHashes().get().getSha256());
     assertEquals(
         "176e9e710ddddd1b357a7d7970831bae59763395a0c18976110cbd35b25e5412dc50f356ec421a7a30265670cf7aec9ed84ee944ba700ec2394b9c876645b960",
-        rekorSnapshot.getHashes().getSha512());
-    assertEquals(797, rekorSnapshot.getLength());
+        rekorSnapshot.getHashes().get().getSha512());
+    assertEquals(797, rekorSnapshot.getLength().get());
     assertEquals(3, rekorSnapshot.getVersion());
   }
 

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/no-size-no-hash-snapshot-timestamp/2.root.json

@@ -0,0 +1 @@
+{"signed":{"_type":"root","spec_version":"1.0","version":2,"expires":"2023-05-13T14:35:58Z","keys":{"0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKzH3HI+8f9hYlrwNynmWtYrdp7kT\n5B13ZcaQJd2gbMw3MXUwAMWksxAjNXXXselrztKQLKEJkj0CRPiXFhtdWg==\n-----END PUBLIC KEY-----\n"}},"7aecf5f0720acfb4fa873896ba05a2d8914f5b6ca90d26ac8bc0f1e491378740":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEs1Stkp5CNyERUPWDa9KF47KjECsx\noobAYi8NUUh5+0Rl34nYR3Y/2IQWu8l2pi9f73Qqsq3kk1cGQMCKRJu1wA==\n-----END PUBLIC KEY-----\n"}},"9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJsV+S1syZdtx5HjiFN5YqRAqD2By\n4R0xDtXptW+UJlJQdfQCGAHvqtpac0edkcWVREhktEqIMbCaYSd75E/JRA==\n-----END PUBLIC KEY-----\n"}},"a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEoZaB1Hu8VvuqgHvwX1mAITts2Zi\ntHhs3suizfA/XDmetnA9BoXhPpLmPJ1n+47xr4Gdr5mcrBzLbM+WcXIs9Q==\n-----END PUBLIC KEY-----\n"}},"a9c5c80b93210eeb34e6264b4b261ff6899d4dbfb8e308f8546722a2bae30687":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEbGNtqWi9Xu7romi12qG+fHYj4SCp\nUCKAOJxXKagVyQNlS6TdJCMHWOJ+0BReT1lQsw6J/SMtc9a5J6Vj7fksBw==\n-----END PUBLIC KEY-----\n"}},"fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfcbhZ0zElnB5dqJBzKiVlofRXBh/\n2snZw32WDcUvl3+7UEtRvmTGZSaAxYCGmAc1EO2j5MGk5wkNkuwiVesd0g==\n-----END PUBLIC KEY-----\n"}}},"roles":{"root":{"keyids":["fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6","0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c","a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40"],"threshold":2},"snapshot":{"keyids":["9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a"],"threshold":1},"targets":{"keyids":["a9c5c80b93210eeb34e6264b4b261ff6899d4dbfb8e308f8546722a2bae30687"],"threshold":1},"timestamp":{"keyids":["7aecf5f0720acfb4fa873896ba05a2d8914f5b6ca90d26ac8bc0f1e491378740"],"threshold":1}},"consistent_snapshot":true},"signatures":[{"keyid":"0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c","sig":"304502204ee7d150bbbf40dc641d1a208be4708be14022da6a86883d2c5a7282eda2659802210095a15450c1e63ff20bd5164979007fbea8a7deea68ebba7a67f8cd2901b686ca"},{"keyid":"a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40","sig":"3046022100845e6b95ccf906b7c44e5993384ecca0efefb0ce9495e9d125856ef4640c5906022100fc4ae0c7f5d32dcccb76b87240f8795d176b10497cced966aac4b8e3db71d0fa"},{"keyid":"fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6","sig":"3045022024637aad4a82ec9416527d2bd54255c56b86ff0c1a8a316d0282ce8f0e18d797022100f51cffa088083bc3c76fe0a26746b99bf49a3b19c4692a12133872a477b6f226"}]}

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/no-size-no-hash-snapshot-timestamp/3.snapshot.json

@@ -0,0 +1 @@
+{"signed":{"_type":"snapshot","spec_version":"1.0","version":3,"expires":"2023-02-19T15:37:48Z","meta":{"targets.json":{"version":3}}},"signatures":[{"keyid":"9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a","sig":"30440220356cee8ca30ff061640f3d88a64cd42f6b3cd3b714e6f5e67596ba798e67f9a702204583f6194190c379ebc248753c64141bfcaf37153de86b7d8249afd15aa9efed"}]}

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/no-size-no-hash-snapshot-timestamp/README.md

@@ -0,0 +1,21 @@
+
+```shell
+cp ../test-template/2.root.json .
+cp -R ../root-signing-workspace tmp
+cd tmp
+# remove hashes and length from snapshots and timestamp
+jq -rc '.signed.meta."targets.json" |= del(.length, .hashes)' repository/snapshot.json | sponge repository/snapshot.json
+jq -rc '.signed.meta."snapshot.json" |= del(.length, .hashes)' repository/timestamp.json | sponge repository/timestamp.json
+# get valid sigs on the new snapshot metadata.
+tuf payload snapshot.json > payload.snapshot.json
+tuf sign-payload --role=snapshot payload.snapshot.json > snapshot.sigs
+tuf add-signatures --signatures snapshot.sigs snapshot.json
+cp staged/snapshot.json ../3.snapshot.json
+# get valid sigs on the new timestamps metadata.
+tuf payload timestamp.json > payload.timestamp.json
+tuf sign-payload --role=timestamp payload.timestamp.json > timestamp.sigs
+tuf add-signatures --signatures timestamp.sigs timestamp.json
+cp staged/timestamp.json ../timestamp.json
+cd ..
+rm -rf tmp
+```

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/no-size-no-hash-snapshot-timestamp/timestamp.json

@@ -0,0 +1 @@
+{"signed":{"_type":"timestamp","spec_version":"1.0","version":3,"expires":"2023-02-13T15:37:48Z","meta":{"snapshot.json":{"version":3}}},"signatures":[{"keyid":"7aecf5f0720acfb4fa873896ba05a2d8914f5b6ca90d26ac8bc0f1e491378740","sig":"3044022060e8b160acae47d6ecc249881c5e7b7beb790e05c519f139bf6c2c98e39cbe54022001a19e057697e5c5023911bc64616f462aed4db0424b6c784c5c03a0fb968fe6"}]}

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/root-signing-workspace/README.md

@@ -1,11 +1,15 @@
 
 # TUF repo creation steps
 
-You'll need the TUF cli to run these commands. 
+You'll need the TUF cli to run these commands. This was generated pre v0.6.0,
+so use v0.5.2 for compatibility till this all this test data is upgraded
 ```shell
-go install github.com/theupdateframework/go-tuf/cmd/tuf@latest
+go install github.com/theupdateframework/go-tuf/cmd/tuf@v0.5.2
 ```
 
+if you are using the repo in root-signing-workspace, you do not need to regenerate
+the repository, you can work within this repo
+
 ```shell
 mkdir root-signing-workspace
 cd root-signing-workspace

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/tmp/2.root.json

@@ -0,0 +1 @@
+{"signed":{"_type":"root","spec_version":"1.0","version":2,"expires":"2023-05-13T14:35:58Z","keys":{"0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKzH3HI+8f9hYlrwNynmWtYrdp7kT\n5B13ZcaQJd2gbMw3MXUwAMWksxAjNXXXselrztKQLKEJkj0CRPiXFhtdWg==\n-----END PUBLIC KEY-----\n"}},"7aecf5f0720acfb4fa873896ba05a2d8914f5b6ca90d26ac8bc0f1e491378740":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEs1Stkp5CNyERUPWDa9KF47KjECsx\noobAYi8NUUh5+0Rl34nYR3Y/2IQWu8l2pi9f73Qqsq3kk1cGQMCKRJu1wA==\n-----END PUBLIC KEY-----\n"}},"9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJsV+S1syZdtx5HjiFN5YqRAqD2By\n4R0xDtXptW+UJlJQdfQCGAHvqtpac0edkcWVREhktEqIMbCaYSd75E/JRA==\n-----END PUBLIC KEY-----\n"}},"a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEoZaB1Hu8VvuqgHvwX1mAITts2Zi\ntHhs3suizfA/XDmetnA9BoXhPpLmPJ1n+47xr4Gdr5mcrBzLbM+WcXIs9Q==\n-----END PUBLIC KEY-----\n"}},"a9c5c80b93210eeb34e6264b4b261ff6899d4dbfb8e308f8546722a2bae30687":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEbGNtqWi9Xu7romi12qG+fHYj4SCp\nUCKAOJxXKagVyQNlS6TdJCMHWOJ+0BReT1lQsw6J/SMtc9a5J6Vj7fksBw==\n-----END PUBLIC KEY-----\n"}},"fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6":{"keytype":"ecdsa-sha2-nistp256","scheme":"ecdsa-sha2-nistp256","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfcbhZ0zElnB5dqJBzKiVlofRXBh/\n2snZw32WDcUvl3+7UEtRvmTGZSaAxYCGmAc1EO2j5MGk5wkNkuwiVesd0g==\n-----END PUBLIC KEY-----\n"}}},"roles":{"root":{"keyids":["fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6","0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c","a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40"],"threshold":2},"snapshot":{"keyids":["9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a"],"threshold":1},"targets":{"keyids":["a9c5c80b93210eeb34e6264b4b261ff6899d4dbfb8e308f8546722a2bae30687"],"threshold":1},"timestamp":{"keyids":["7aecf5f0720acfb4fa873896ba05a2d8914f5b6ca90d26ac8bc0f1e491378740"],"threshold":1}},"consistent_snapshot":true},"signatures":[{"keyid":"0b5108e406f6d2f59ef767797b314be99d35903950ba43a2d51216eeeb8da98c","sig":"304502204ee7d150bbbf40dc641d1a208be4708be14022da6a86883d2c5a7282eda2659802210095a15450c1e63ff20bd5164979007fbea8a7deea68ebba7a67f8cd2901b686ca"},{"keyid":"a041140325d05d8a7643d5649a8c4296f8e6b020fb73bf83c52319b1a7230a40","sig":"3046022100845e6b95ccf906b7c44e5993384ecca0efefb0ce9495e9d125856ef4640c5906022100fc4ae0c7f5d32dcccb76b87240f8795d176b10497cced966aac4b8e3db71d0fa"},{"keyid":"fca39ff47a3a91605f2c56501e84b4fe3b9a66b96a022275e866bd19353f93e6","sig":"3045022024637aad4a82ec9416527d2bd54255c56b86ff0c1a8a316d0282ce8f0e18d797022100f51cffa088083bc3c76fe0a26746b99bf49a3b19c4692a12133872a477b6f226"}]}

sigstore-java/src/test/resources/dev/sigstore/tuf/synthetic/tmp/3.snapshot.json

@@ -0,0 +1 @@
+{"signed":{"_type":"snapshot","spec_version":"1.0","version":3,"expires":"2023-02-19T15:37:48Z","meta":{"targets.json":{"version":3}}},"signatures":[{"keyid":"9354bd3deaa572ed06306ddfad457037918534ece677cf962526a6fd40112d7a","sig":"3046022100a894577e33d9f0771a15f14ea4932daaf31fa86c8a2d6fd2f6b22042a8673722022100a12583f21fb237627dee73a7c5a101006b64e67894b93b596d491aecab21c3d5"}]}