From 3fd9b032fa04213473c933d22a447d36b3a8d51b Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Sun, 15 Sep 2024 18:19:45 +0800 Subject: [PATCH] Print a JSON representation of part of the header [#63] (#159) Print a JSON representation of part of the header [#63] * refactor show to take io.Writer instead of os.Stdout * add integration test for Show command * hide write command as it's not implemented yet. --- .github/workflows/test.yml | 2 +- .gitignore | 4 +- main.go | 21 +++++--- pmtiles/directory.go | 62 +++++++++++++++++++++--- pmtiles/directory_test.go | 36 ++++++++++++++ pmtiles/extract_test.go | 2 - pmtiles/fixtures/test_fixture_1.pmtiles | Bin 0 -> 468 bytes pmtiles/show.go | 14 +++--- pmtiles/show_test.go | 33 +++++++++++++ pmtiles/write.go | 42 ++++++++++++++++ pmtiles/write_test.go | 19 ++++++++ 11 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 pmtiles/fixtures/test_fixture_1.pmtiles create mode 100644 pmtiles/show_test.go create mode 100644 pmtiles/write.go create mode 100644 pmtiles/write_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b226b80..5739e7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,6 @@ jobs: - run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi - run: go vet caddy/pmtiles_proxy.go - run: go vet main.go - - run: go vet pmtiles/* + - run: go vet ./pmtiles - name: Run Revive Action by pulling pre-built image uses: docker://morphy/revive-action:v2 diff --git a/.gitignore b/.gitignore index f93b9dc..8fc98d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ dist/ go-pmtiles -*.pmtiles +/*.pmtiles +/*.mbtiles +*.json *.geojson *.tsv.gz diff --git a/main.go b/main.go index bff9e06..85f441c 100644 --- a/main.go +++ b/main.go @@ -36,11 +36,12 @@ var cli struct { } `cmd:"" help:"Convert an MBTiles or older spec version to PMTiles."` Show struct { - Path string `arg:""` - Bucket string `help:"Remote bucket"` - Metadata bool `help:"Print only the JSON metadata."` - Tilejson bool `help:"Print the TileJSON."` - PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` + Path string `arg:""` + Bucket string `help:"Remote bucket"` + Metadata bool `help:"Print only the JSON metadata."` + HeaderJson bool `help:"Print a JSON representation of the header information."` + Tilejson bool `help:"Print the TileJSON."` + PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` } `cmd:"" help:"Inspect a local or remote archive."` Tile struct { @@ -51,6 +52,12 @@ var cli struct { Bucket string `help:"Remote bucket"` } `cmd:"" help:"Fetch one tile from a local or remote archive and output on stdout."` + Write struct { + Input string `arg:"" help:"Input archive file." type:"existingfile"` + HeaderJson string `help:"Input header JSON file (written by show --header-json)." type:"existingfile"` + Metadata string `help:"Input metadata JSON (written by show --metadata)." type:"existingfile"` + } `cmd:"" help:"Write header data or metadata to an existing archive." hidden:""` + Extract struct { Input string `arg:"" help:"Input local or remote archive."` Output string `arg:"" help:"Output archive." type:"path"` @@ -122,12 +129,12 @@ func main() { switch ctx.Command() { case "show ": - err := pmtiles.Show(logger, cli.Show.Bucket, cli.Show.Path, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0) + err := pmtiles.Show(logger, os.Stdout, cli.Show.Bucket, cli.Show.Path, cli.Show.HeaderJson, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0) if err != nil { logger.Fatalf("Failed to show archive, %v", err) } case "tile ": - err := pmtiles.Show(logger, cli.Tile.Bucket, cli.Tile.Path, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y) + err := pmtiles.Show(logger, os.Stdout, cli.Tile.Bucket, cli.Tile.Path, false, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y) if err != nil { logger.Fatalf("Failed to show tile, %v", err) } diff --git a/pmtiles/directory.go b/pmtiles/directory.go index 5820810..26cf286 100644 --- a/pmtiles/directory.go +++ b/pmtiles/directory.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "encoding/binary" + "encoding/json" "fmt" ) @@ -63,6 +64,23 @@ type HeaderV3 struct { CenterLatE7 int32 } +// HeaderJson is a human-readable representation of parts of the binary header +// that may need to be manually edited. +// Omitted parts are the responsibility of the generator program and not editable. +type HeaderJson struct { + TileCompression string + TileType string + MinZoom int + MaxZoom int + MinLon float64 + MinLat float64 + MaxLon float64 + MaxLat float64 + CenterZoom int + CenterLon float64 + CenterLat float64 +} + func headerContentType(header HeaderV3) (string, bool) { switch header.TileType { case Mvt: @@ -80,23 +98,31 @@ func headerContentType(header HeaderV3) (string, bool) { } } -func headerExt(header HeaderV3) string { - switch header.TileType { +func stringifiedTileType(t TileType) string { + switch t { case Mvt: - return ".mvt" + return "mvt" case Png: - return ".png" + return "png" case Jpeg: - return ".jpg" + return "jpg" case Webp: - return ".webp" + return "webp" case Avif: - return ".avif" + return "avif" default: return "" } } +func headerExt(header HeaderV3) string { + base := stringifiedTileType(header.TileType) + if base == "" { + return "" + } + return "." + base +} + func headerContentEncoding(compression Compression) (string, bool) { switch compression { case Gzip: @@ -108,6 +134,28 @@ func headerContentEncoding(compression Compression) (string, bool) { } } +func headerToJson(header HeaderV3) HeaderJson { + compressionString, _ := headerContentEncoding(header.TileCompression) + return HeaderJson{ + TileCompression: compressionString, + TileType: stringifiedTileType(header.TileType), + MinZoom: int(header.MinZoom), + MaxZoom: int(header.MaxZoom), + MinLon: float64(header.MinLonE7) / 10000000, + MinLat: float64(header.MinLatE7) / 10000000, + MaxLon: float64(header.MaxLonE7) / 10000000, + MaxLat: float64(header.MaxLatE7) / 10000000, + CenterZoom: int(header.CenterZoom), + CenterLon: float64(header.CenterLonE7) / 10000000, + CenterLat: float64(header.CenterLatE7) / 10000000, + } +} + +func headerToStringifiedJson(header HeaderV3) string { + s, _ := json.MarshalIndent(headerToJson(header), "", " ") + return string(s) +} + // EntryV3 is an entry in a PMTiles spec version 3 directory. type EntryV3 struct { TileID uint64 diff --git a/pmtiles/directory_test.go b/pmtiles/directory_test.go index a5a8399..21b06ee 100644 --- a/pmtiles/directory_test.go +++ b/pmtiles/directory_test.go @@ -84,6 +84,33 @@ func TestHeaderRoundtrip(t *testing.T) { assert.Equal(t, int32(32000000), result.CenterLatE7) } +func TestHeaderJsonRoundtrip(t *testing.T) { + header := HeaderV3{} + header.TileCompression = Brotli + header.TileType = Mvt + header.MinZoom = 1 + header.MaxZoom = 3 + header.MinLonE7 = 1.1 * 10000000 + header.MinLatE7 = 2.1 * 10000000 + header.MaxLonE7 = 1.2 * 10000000 + header.MaxLatE7 = 2.2 * 10000000 + header.CenterZoom = 2 + header.CenterLonE7 = 3.1 * 10000000 + header.CenterLatE7 = 3.2 * 10000000 + j := headerToJson(header) + assert.Equal(t, "br", j.TileCompression) + assert.Equal(t, "mvt", j.TileType) + assert.Equal(t, 1, j.MinZoom) + assert.Equal(t, 3, j.MaxZoom) + assert.Equal(t, 2, j.CenterZoom) + assert.Equal(t, 1.1, j.MinLon) + assert.Equal(t, 2.1, j.MinLat) + assert.Equal(t, 1.2, j.MaxLon) + assert.Equal(t, 2.2, j.MaxLat) + assert.Equal(t, 3.1, j.CenterLon) + assert.Equal(t, 3.2, j.CenterLat) +} + func TestOptimizeDirectories(t *testing.T) { rand.Seed(3857) entries := make([]EntryV3, 0) @@ -171,3 +198,12 @@ func TestBuildRootsLeaves(t *testing.T) { _, _, numLeaves := buildRootsLeaves(entries, 1) assert.Equal(t, 1, numLeaves) } + +func TestStringifiedExtension(t *testing.T) { + assert.Equal(t, "", headerExt(HeaderV3{})) + assert.Equal(t, ".mvt", headerExt(HeaderV3{TileType: Mvt})) + assert.Equal(t, ".png", headerExt(HeaderV3{TileType: Png})) + assert.Equal(t, ".jpg", headerExt(HeaderV3{TileType: Jpeg})) + assert.Equal(t, ".webp", headerExt(HeaderV3{TileType: Webp})) + assert.Equal(t, ".avif", headerExt(HeaderV3{TileType: Avif})) +} diff --git a/pmtiles/extract_test.go b/pmtiles/extract_test.go index 2801041..d3d14ae 100644 --- a/pmtiles/extract_test.go +++ b/pmtiles/extract_test.go @@ -1,7 +1,6 @@ package pmtiles import ( - "fmt" "github.com/RoaringBitmap/roaring/roaring64" "github.com/stretchr/testify/assert" "testing" @@ -164,5 +163,4 @@ func TestMergeRangesMultiple(t *testing.T) { assert.Equal(t, 1, result.Len()) assert.Equal(t, srcDstRange{0, 0, 90}, front.Rng) assert.Equal(t, 3, len(front.CopyDiscards)) - fmt.Println(result) } diff --git a/pmtiles/fixtures/test_fixture_1.pmtiles b/pmtiles/fixtures/test_fixture_1.pmtiles new file mode 100644 index 0000000000000000000000000000000000000000..c86db1f27bba6d3818bb5da5ac6b78e8f6e3303a GIT binary patch literal 468 zcmWIW4av+&EoQD~fB;D-Jp)RAhtlX&KO;oM6)FMcGN988OiVB#pvwAbGZ-3x7!KsS zIlyct;pCKrl2jH3ANx)QR-h0r{uR(?Y;E0N${r@wV$Rqz?{?h$%CZ~pR-fD;chPxcGvD5#swCfUJ3q9X zyYJfGQaN*z>CwX)(_MHoE_$j>pZmpR;iPE+bNH@ihyD-PGDW#IC`?~Ew^Gu@^U;5~ zMLN|*T=}&jpE@sYjM(y1WHrOxzp@|qI{uNHt+DuR>j{?#b(QndSF&YxKiSJB=o0K; zmUMDvv&>(=%}+VLv)Zh>lkdN)q3RE>2ir|u&wq}OjgFMDdZaDyH+~%MwD)0=m+&ls zb+dE0Za)PE6vEq+FRf2_B&6%9tJmp&dX0vru2-+Q&I#R