diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b226b80..5739e7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,6 @@ jobs: - run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi - run: go vet caddy/pmtiles_proxy.go - run: go vet main.go - - run: go vet pmtiles/* + - run: go vet ./pmtiles - name: Run Revive Action by pulling pre-built image uses: docker://morphy/revive-action:v2 diff --git a/.gitignore b/.gitignore index f93b9dc..8fc98d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ dist/ go-pmtiles -*.pmtiles +/*.pmtiles +/*.mbtiles +*.json *.geojson *.tsv.gz diff --git a/main.go b/main.go index bff9e06..85f441c 100644 --- a/main.go +++ b/main.go @@ -36,11 +36,12 @@ var cli struct { } `cmd:"" help:"Convert an MBTiles or older spec version to PMTiles."` Show struct { - Path string `arg:""` - Bucket string `help:"Remote bucket"` - Metadata bool `help:"Print only the JSON metadata."` - Tilejson bool `help:"Print the TileJSON."` - PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` + Path string `arg:""` + Bucket string `help:"Remote bucket"` + Metadata bool `help:"Print only the JSON metadata."` + HeaderJson bool `help:"Print a JSON representation of the header information."` + Tilejson bool `help:"Print the TileJSON."` + PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"` } `cmd:"" help:"Inspect a local or remote archive."` Tile struct { @@ -51,6 +52,12 @@ var cli struct { Bucket string `help:"Remote bucket"` } `cmd:"" help:"Fetch one tile from a local or remote archive and output on stdout."` + Write struct { + Input string `arg:"" help:"Input archive file." type:"existingfile"` + HeaderJson string `help:"Input header JSON file (written by show --header-json)." type:"existingfile"` + Metadata string `help:"Input metadata JSON (written by show --metadata)." type:"existingfile"` + } `cmd:"" help:"Write header data or metadata to an existing archive." hidden:""` + Extract struct { Input string `arg:"" help:"Input local or remote archive."` Output string `arg:"" help:"Output archive." type:"path"` @@ -122,12 +129,12 @@ func main() { switch ctx.Command() { case "show ": - err := pmtiles.Show(logger, cli.Show.Bucket, cli.Show.Path, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0) + err := pmtiles.Show(logger, os.Stdout, cli.Show.Bucket, cli.Show.Path, cli.Show.HeaderJson, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0) if err != nil { logger.Fatalf("Failed to show archive, %v", err) } case "tile ": - err := pmtiles.Show(logger, cli.Tile.Bucket, cli.Tile.Path, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y) + err := pmtiles.Show(logger, os.Stdout, cli.Tile.Bucket, cli.Tile.Path, false, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y) if err != nil { logger.Fatalf("Failed to show tile, %v", err) } diff --git a/pmtiles/directory.go b/pmtiles/directory.go index 5820810..26cf286 100644 --- a/pmtiles/directory.go +++ b/pmtiles/directory.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "encoding/binary" + "encoding/json" "fmt" ) @@ -63,6 +64,23 @@ type HeaderV3 struct { CenterLatE7 int32 } +// HeaderJson is a human-readable representation of parts of the binary header +// that may need to be manually edited. +// Omitted parts are the responsibility of the generator program and not editable. +type HeaderJson struct { + TileCompression string + TileType string + MinZoom int + MaxZoom int + MinLon float64 + MinLat float64 + MaxLon float64 + MaxLat float64 + CenterZoom int + CenterLon float64 + CenterLat float64 +} + func headerContentType(header HeaderV3) (string, bool) { switch header.TileType { case Mvt: @@ -80,23 +98,31 @@ func headerContentType(header HeaderV3) (string, bool) { } } -func headerExt(header HeaderV3) string { - switch header.TileType { +func stringifiedTileType(t TileType) string { + switch t { case Mvt: - return ".mvt" + return "mvt" case Png: - return ".png" + return "png" case Jpeg: - return ".jpg" + return "jpg" case Webp: - return ".webp" + return "webp" case Avif: - return ".avif" + return "avif" default: return "" } } +func headerExt(header HeaderV3) string { + base := stringifiedTileType(header.TileType) + if base == "" { + return "" + } + return "." + base +} + func headerContentEncoding(compression Compression) (string, bool) { switch compression { case Gzip: @@ -108,6 +134,28 @@ func headerContentEncoding(compression Compression) (string, bool) { } } +func headerToJson(header HeaderV3) HeaderJson { + compressionString, _ := headerContentEncoding(header.TileCompression) + return HeaderJson{ + TileCompression: compressionString, + TileType: stringifiedTileType(header.TileType), + MinZoom: int(header.MinZoom), + MaxZoom: int(header.MaxZoom), + MinLon: float64(header.MinLonE7) / 10000000, + MinLat: float64(header.MinLatE7) / 10000000, + MaxLon: float64(header.MaxLonE7) / 10000000, + MaxLat: float64(header.MaxLatE7) / 10000000, + CenterZoom: int(header.CenterZoom), + CenterLon: float64(header.CenterLonE7) / 10000000, + CenterLat: float64(header.CenterLatE7) / 10000000, + } +} + +func headerToStringifiedJson(header HeaderV3) string { + s, _ := json.MarshalIndent(headerToJson(header), "", " ") + return string(s) +} + // EntryV3 is an entry in a PMTiles spec version 3 directory. type EntryV3 struct { TileID uint64 diff --git a/pmtiles/directory_test.go b/pmtiles/directory_test.go index a5a8399..21b06ee 100644 --- a/pmtiles/directory_test.go +++ b/pmtiles/directory_test.go @@ -84,6 +84,33 @@ func TestHeaderRoundtrip(t *testing.T) { assert.Equal(t, int32(32000000), result.CenterLatE7) } +func TestHeaderJsonRoundtrip(t *testing.T) { + header := HeaderV3{} + header.TileCompression = Brotli + header.TileType = Mvt + header.MinZoom = 1 + header.MaxZoom = 3 + header.MinLonE7 = 1.1 * 10000000 + header.MinLatE7 = 2.1 * 10000000 + header.MaxLonE7 = 1.2 * 10000000 + header.MaxLatE7 = 2.2 * 10000000 + header.CenterZoom = 2 + header.CenterLonE7 = 3.1 * 10000000 + header.CenterLatE7 = 3.2 * 10000000 + j := headerToJson(header) + assert.Equal(t, "br", j.TileCompression) + assert.Equal(t, "mvt", j.TileType) + assert.Equal(t, 1, j.MinZoom) + assert.Equal(t, 3, j.MaxZoom) + assert.Equal(t, 2, j.CenterZoom) + assert.Equal(t, 1.1, j.MinLon) + assert.Equal(t, 2.1, j.MinLat) + assert.Equal(t, 1.2, j.MaxLon) + assert.Equal(t, 2.2, j.MaxLat) + assert.Equal(t, 3.1, j.CenterLon) + assert.Equal(t, 3.2, j.CenterLat) +} + func TestOptimizeDirectories(t *testing.T) { rand.Seed(3857) entries := make([]EntryV3, 0) @@ -171,3 +198,12 @@ func TestBuildRootsLeaves(t *testing.T) { _, _, numLeaves := buildRootsLeaves(entries, 1) assert.Equal(t, 1, numLeaves) } + +func TestStringifiedExtension(t *testing.T) { + assert.Equal(t, "", headerExt(HeaderV3{})) + assert.Equal(t, ".mvt", headerExt(HeaderV3{TileType: Mvt})) + assert.Equal(t, ".png", headerExt(HeaderV3{TileType: Png})) + assert.Equal(t, ".jpg", headerExt(HeaderV3{TileType: Jpeg})) + assert.Equal(t, ".webp", headerExt(HeaderV3{TileType: Webp})) + assert.Equal(t, ".avif", headerExt(HeaderV3{TileType: Avif})) +} diff --git a/pmtiles/extract_test.go b/pmtiles/extract_test.go index 2801041..d3d14ae 100644 --- a/pmtiles/extract_test.go +++ b/pmtiles/extract_test.go @@ -1,7 +1,6 @@ package pmtiles import ( - "fmt" "github.com/RoaringBitmap/roaring/roaring64" "github.com/stretchr/testify/assert" "testing" @@ -164,5 +163,4 @@ func TestMergeRangesMultiple(t *testing.T) { assert.Equal(t, 1, result.Len()) assert.Equal(t, srcDstRange{0, 0, 90}, front.Rng) assert.Equal(t, 3, len(front.CopyDiscards)) - fmt.Println(result) } diff --git a/pmtiles/fixtures/test_fixture_1.pmtiles b/pmtiles/fixtures/test_fixture_1.pmtiles new file mode 100644 index 0000000..c86db1f Binary files /dev/null and b/pmtiles/fixtures/test_fixture_1.pmtiles differ diff --git a/pmtiles/show.go b/pmtiles/show.go index dc6e3a9..09318d4 100644 --- a/pmtiles/show.go +++ b/pmtiles/show.go @@ -14,7 +14,7 @@ import ( ) // Show prints detailed information about an archive. -func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, showTilejson bool, publicURL string, showTile bool, z int, x int, y int) error { +func Show(_ *log.Logger, output io.Writer, bucketURL string, key string, showHeaderJsonOnly bool, showMetadataOnly bool, showTilejson bool, publicURL string, showTile bool, z int, x int, y int) error { ctx := context.Background() bucketURL, key, err := NormalizeBucketKey(bucketURL, "", key) @@ -90,11 +90,13 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh metadataReader.Close() if showMetadataOnly && showTilejson { - return fmt.Errorf("cannot use --metadata and --tilejson together") + return fmt.Errorf("cannot use more than one of --header-json, --metadata, and --tilejson together") } - if showMetadataOnly { - fmt.Print(string(metadataBytes)) + if showHeaderJsonOnly { + fmt.Fprintln(output, headerToStringifiedJson(header)) + } else if showMetadataOnly { + fmt.Fprintln(output, string(metadataBytes)) } else if showTilejson { if publicURL == "" { // Using Fprintf instead of logger here, as this message should be written to Stderr in case @@ -105,7 +107,7 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh if err != nil { return fmt.Errorf("Failed to create tilejson for %s, %w", key, err) } - fmt.Print(string(tilejsonBytes)) + fmt.Fprintln(output, string(tilejsonBytes)) } else { fmt.Printf("pmtiles spec version: %d\n", header.SpecVersion) // fmt.Printf("total size: %s\n", humanize.Bytes(uint64(r.Size()))) @@ -164,7 +166,7 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh if err != nil { return fmt.Errorf("I/O Error") } - os.Stdout.Write(tileBytes) + output.Write(tileBytes) break } dirOffset = header.LeafDirectoryOffset + entry.Offset diff --git a/pmtiles/show_test.go b/pmtiles/show_test.go new file mode 100644 index 0000000..3ea59c0 --- /dev/null +++ b/pmtiles/show_test.go @@ -0,0 +1,33 @@ +package pmtiles + +import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/assert" + "log" + "os" + "testing" +) + +func TestShowHeader(t *testing.T) { + var b bytes.Buffer + logger := log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile) + err := Show(logger, &b, "", "fixtures/test_fixture_1.pmtiles", true, false, false, "", false, 0, 0, 0) + assert.Nil(t, err) + + var input map[string]interface{} + json.Unmarshal(b.Bytes(), &input) + assert.Equal(t, "mvt", input["TileType"]) + assert.Equal(t, "gzip", input["TileCompression"]) +} + +func TestShowMetadata(t *testing.T) { + var b bytes.Buffer + logger := log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile) + err := Show(logger, &b, "", "fixtures/test_fixture_1.pmtiles", false, true, false, "", false, 0, 0, 0) + assert.Nil(t, err) + + var input map[string]interface{} + json.Unmarshal(b.Bytes(), &input) + assert.Equal(t, "tippecanoe v2.5.0", input["generator"]) +} diff --git a/pmtiles/write.go b/pmtiles/write.go new file mode 100644 index 0000000..a4e0f1e --- /dev/null +++ b/pmtiles/write.go @@ -0,0 +1,42 @@ +package pmtiles + +import ( + "fmt" + "log" + "os" +) + +func Write(logger *log.Logger, inputArchive string, newHeaderJsonFile string, newMetadataFile string) error { + if newMetadataFile == "" { + if newHeaderJsonFile == "" { + return fmt.Errorf("No data to write.") + } + // we can write the header in-place without writing the whole file. + return nil + } + + // write metadata: + // always writes in this order: + // copy the header + // copy the root directory + // write the new the metadata + // copy the leaf directories + // copy the tile data + file, err := os.OpenFile(inputArchive, os.O_RDWR, 0666) + + buf := make([]byte, 127) + _, err = file.Read(buf) + if err != nil { + return err + } + originalHeader, _ := deserializeHeader(buf) + + // modify the header + + buf = serializeHeader(originalHeader) + _, err = file.WriteAt(buf, 0) + if err != nil { + return err + } + return nil +} diff --git a/pmtiles/write_test.go b/pmtiles/write_test.go new file mode 100644 index 0000000..aa227c6 --- /dev/null +++ b/pmtiles/write_test.go @@ -0,0 +1,19 @@ +package pmtiles + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestWriteHeader(t *testing.T) { + tempDir, _ := ioutil.TempDir("", "testing") + defer os.RemoveAll(tempDir) + src, _ := os.Open("fixtures/test_fixture_1.pmtiles") + defer src.Close() + dest, _ := os.Create(filepath.Join(tempDir, "test.pmtiles")) + defer dest.Close() + _, _ = io.Copy(dest, src) +}