Parse generation from Storage URI

gcs-fetcher/pkg/common/common.go

@@ -18,6 +18,7 @@ package common
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 )
 
@@ -38,26 +39,53 @@ type ManifestItem struct {
 //
 // It supports URIs in either of these forms:
 // - https://storage.googleapis.com/bucket/path/to/object
+// - https://storage.googleapis.com/bucket/path/to/object#1234
 // - gs://bucket/path/to/object
+// - gs://bucket/path/to/object#1234
 //
-// In both of the above cases, bucket=bucket and object=path/to/object.
+// In the above cases bucket=bucket, object=path/to/object, and when specified generation=1234.
 func ParseBucketObject(uri string) (bucket, object string, generation int64, err error) {
-	// TODO: Parse generation.
 	switch {
 	case strings.HasPrefix(uri, "https://storage.googleapis.com/") || strings.HasPrefix(uri, "http://storage.googleapis.com/"):
 		// uri looks like "https://storage.googleapis.com/staging.my-project.appspot.com/3aa080e5e72a610b06033dbfee288483d87cfd61"
 		if parts := strings.Split(uri, "/"); len(parts) >= 5 {
 			bucket := parts[3]
-			object := strings.Join(parts[4:], "/")
-			return bucket, object, 0, nil
+			object, generation, err := splitObjectAndGeneration(strings.Join(parts[4:], "/"))
+			if err != nil {
+				return "", "", 0, fmt.Errorf("cannot parse object/generation from uri %q", uri)
+			}
+			return bucket, object, generation, nil
 		}
 	case strings.HasPrefix(uri, "gs://"):
 		// uri looks like "gs://my-bucket/manifest-20171004T175409.json"
 		if parts := strings.Split(uri, "/"); len(parts) >= 4 {
 			bucket := parts[2]
-			object := strings.Join(parts[3:], "/")
-			return bucket, object, 0, nil
+			object, generation, err := splitObjectAndGeneration(strings.Join(parts[3:], "/"))
+			if err != nil {
+				return "", "", 0, fmt.Errorf("cannot parse object/generation from uri %q", uri)
+			}
+
+			return bucket, object, generation, nil
 		}
 	}
 	return "", "", 0, fmt.Errorf("cannot parse bucket/object from uri %q", uri)
 }
+
+func splitObjectAndGeneration(fullObject string) (object string, generation int64, err error) {
+	generation = 0
+	object = fullObject
+
+	generationIndex := strings.LastIndex(fullObject, "#")
+	// if generation exists parse it
+	// e.g. myFile.json#123456 is 123456
+	if generationIndex > 0 {
+		generation, err = strconv.ParseInt(fullObject[generationIndex + 1:], 10, 64)
+		if err != nil {
+			return "", 0, err
+		}
+
+		object = fullObject[:generationIndex]
+	}
+
+	return object, generation, nil
+}

gcs-fetcher/pkg/common/common_test.go

@@ -21,10 +21,11 @@ import (
 
 func TestParseBucketObject(t *testing.T) {
 	for _, c := range []struct {
-		uri     string
-		bucket  string
-		object  string
-		wantErr bool
+		uri        string
+		bucket     string
+		object     string
+		generation int64
+		wantErr    bool
 	}{{
 		uri:    "https://storage.googleapis.com/staging.appid.appspot.com/abc123",
 		bucket: "staging.appid.appspot.com",
@@ -37,12 +38,25 @@ func TestParseBucketObject(t *testing.T) {
 		uri:    "https://storage.googleapis.com/some-bucket/abc123",
 		bucket: "some-bucket",
 		object: "abc123",
+	}, {
+		uri:        "https://storage.googleapis.com/some-bucket/abc123#4444",
+		bucket:     "some-bucket",
+		object:     "abc123",
+		generation: 4444,
+	}, {
+		uri:        "https://storage.googleapis.com/some-bucket/myFile#2.txt#4444",
+		bucket:     "some-bucket",
+		object:     "myFile#2.txt",
+		generation: 4444,
 	}, {
 		uri:     "https://storage.googleapis.com/too-short",
 		wantErr: true,
 	}, {
 		uri:     "https://incorrect-domain.com/some-bucket.google.com.a.appspot.com/some/path",
 		wantErr: true,
+	}, {
+		uri:     "https://storage.googleapis.com/some-bucket/abc123#invalidGeneration444",
+		wantErr: true,
 	}, {
 		uri:    "gs://my-bucket/manifest-20171004T175409.json",
 		bucket: "my-bucket",
@@ -59,6 +73,16 @@ func TestParseBucketObject(t *testing.T) {
 		uri:    "gs://some-bucket/abc123",
 		bucket: "some-bucket",
 		object: "abc123",
+	}, {
+		uri:        "gs://some-bucket/abc123#4444",
+		bucket:     "some-bucket",
+		object:     "abc123",
+		generation: 4444,
+	}, {
+		uri:        "gs://some-bucket/myFile#2.txt#4444",
+		bucket:     "some-bucket",
+		object:     "myFile#2.txt",
+		generation: 4444,
 	}, {
 		uri:    "http://storage.googleapis.com/my-bucket/test-memchache/server.js",
 		bucket: "my-bucket",
@@ -69,14 +93,17 @@ func TestParseBucketObject(t *testing.T) {
 	}, {
 		uri:     "some-bucket/some/path",
 		wantErr: true,
+	}, {
+		uri:     "gs://some-bucket/abc123#invalidGeneration444",
+		wantErr: true,
 	}} {
-		bucket, object, _, err := ParseBucketObject(c.uri)
+		bucket, object, generation, err := ParseBucketObject(c.uri)
 		if (err != nil) != c.wantErr {
 			t.Errorf("ParseBucketObject(%q): got %v, wantErr = %t", c.uri, err, c.wantErr)
 		}
 		if err == nil {
-			if bucket != c.bucket || object != c.object {
-				t.Errorf("parseBucketObject(%q) = (%q, %q); want (%q, %q)", c.uri, bucket, object, c.bucket, c.object)
+			if bucket != c.bucket || object != c.object || generation != c.generation {
+				t.Errorf("parseBucketObject(%q) = (%q, %q, %d); want (%q, %q, %d)", c.uri, bucket, object, generation, c.bucket, c.object, c.generation)
 			}
 		}
 	}