Edit the metadata and header of a pmtiles archive.

main.go

@@ -1,4 +1,4 @@
 package main
 
 import (
 	"fmt"
@@ -32,14 +32,14 @@
 		Output          string `arg:"" help:"Output PMTiles archive." type:"path"`
 		Force           bool   `help:"Force removal."`
 		NoDeduplication bool   `help:"Don't attempt to deduplicate tiles."`
-		Tmpdir          string `help:"An optional path to a folder for tmp data." type:"existingdir"`
+		Tmpdir          string `help:"An optional path to a folder for temporary files." type:"existingdir"`
 	} `cmd:"" help:"Convert an MBTiles or older spec version to PMTiles."`
 
 	Show struct {
 		Path       string `arg:""`
 		Bucket     string `help:"Remote bucket"`
 		Metadata   bool   `help:"Print only the JSON metadata."`
 		HeaderJson bool   `help:"Print a JSON representation of the header information."`
 		Tilejson   bool   `help:"Print the TileJSON."`
 		PublicURL  string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"`
 	} `cmd:"" help:"Inspect a local or remote archive."`
@@ -52,11 +52,11 @@
 		Bucket string `help:"Remote bucket"`
 	} `cmd:"" help:"Fetch one tile from a local or remote archive and output on stdout."`
 
-	Write struct {
+	Edit struct {
 		Input      string `arg:"" help:"Input archive file." type:"existingfile"`
 		HeaderJson string `help:"Input header JSON file (written by show --header-json)." type:"existingfile"`
 		Metadata   string `help:"Input metadata JSON (written by show --metadata)." type:"existingfile"`
-	} `cmd:"" help:"Write header data or metadata to an existing archive." hidden:""`
+	} `cmd:"" help:"Edit JSON metadata or parts of the header in-place." hidden:""`
 
 	Extract struct {
 		Input           string  `arg:"" help:"Input local or remote archive."`
@@ -211,7 +211,7 @@
 		if err != nil {
 			logger.Fatalf("Failed to convert %s, %v", path, err)
 		}
-	case "upload <inputpmtiles> <remotepmtiles>":
+	case "upload <input-pmtiles> <remote-pmtiles>":
 		err := pmtiles.Upload(logger, cli.Upload.InputPmtiles, cli.Upload.Bucket, cli.Upload.RemotePmtiles, cli.Upload.MaxConcurrency)
 
 		if err != nil {
@@ -222,6 +222,11 @@
 		if err != nil {
 			logger.Fatalf("Failed to verify archive, %v", err)
 		}
+	case "edit <input>":
+		err := pmtiles.Edit(logger, cli.Edit.Input, cli.Edit.HeaderJson, cli.Edit.Metadata)
+		if err != nil {
+			logger.Fatalf("Failed to edit archive, %v", err)
+		}
 	case "makesync <input>":
 		err := pmtiles.Makesync(logger, version, cli.Makesync.Input, cli.Makesync.BlockSizeKb, cli.Makesync.Checksum)
 		if err != nil {

pmtiles/directory.go

@@ -67,18 +67,14 @@ type HeaderV3 struct {
 // HeaderJson is a human-readable representation of parts of the binary header
 // that may need to be manually edited.
 // Omitted parts are the responsibility of the generator program and not editable.
+// The formatting is aligned with the TileJSON / MBTiles specification.
 type HeaderJson struct {
-	TileCompression string
-	TileType        string
-	MinZoom         int
-	MaxZoom         int
-	MinLon          float64
-	MinLat          float64
-	MaxLon          float64
-	MaxLat          float64
-	CenterZoom      int
-	CenterLon       float64
-	CenterLat       float64
+	TileCompression string    `json:"tile_compression"`
+	TileType        string    `json:"tile_type"`
+	MinZoom         int       `json:"minzoom"`
+	MaxZoom         int       `json:"maxzoom"`
+	Bounds          []float64 `json:"bounds"`
+	Center          []float64 `json:"center"`
 }
 
 func headerContentType(header HeaderV3) (string, bool) {
@@ -98,7 +94,7 @@ func headerContentType(header HeaderV3) (string, bool) {
 	}
 }
 
-func stringifiedTileType(t TileType) string {
+func tileTypeToString(t TileType) string {
 	switch t {
 	case Mvt:
 		return "mvt"
@@ -115,39 +111,70 @@ func stringifiedTileType(t TileType) string {
 	}
 }
 
+func stringToTileType(t string) TileType {
+	switch t {
+	case "mvt":
+		return Mvt
+	case "png":
+		return Png
+	case "jpg":
+		return Jpeg
+	case "webp":
+		return Webp
+	case "avif":
+		return Avif
+	default:
+		return UnknownTileType
+	}
+}
+
 func headerExt(header HeaderV3) string {
-	base := stringifiedTileType(header.TileType)
+	base := tileTypeToString(header.TileType)
 	if base == "" {
 		return ""
 	}
 	return "." + base
 }
 
-func headerContentEncoding(compression Compression) (string, bool) {
+func compressionToString(compression Compression) (string, bool) {
 	switch compression {
+	case NoCompression:
+		return "none", false
 	case Gzip:
 		return "gzip", true
 	case Brotli:
 		return "br", true
+	case Zstd:
+		return "zstd", true
 	default:
-		return "", false
+		return "unknown", false
+	}
+}
+
+func stringToCompression(s string) Compression {
+	switch s {
+	case "none":
+		return NoCompression
+	case "gzip":
+		return Gzip
+	case "br":
+		return Brotli
+	case "zstd":
+		return Zstd
+	default:
+		return UnknownCompression
 	}
 }
 
 func headerToJson(header HeaderV3) HeaderJson {
-	compressionString, _ := headerContentEncoding(header.TileCompression)
+	compressionString, _ := compressionToString(header.TileCompression)
 	return HeaderJson{
 		TileCompression: compressionString,
-		TileType:        stringifiedTileType(header.TileType),
+		TileType:        tileTypeToString(header.TileType),
 		MinZoom:         int(header.MinZoom),
 		MaxZoom:         int(header.MaxZoom),
-		MinLon:          float64(header.MinLonE7) / 10000000,
-		MinLat:          float64(header.MinLatE7) / 10000000,
-		MaxLon:          float64(header.MaxLonE7) / 10000000,
-		MaxLat:          float64(header.MaxLatE7) / 10000000,
-		CenterZoom:      int(header.CenterZoom),
-		CenterLon:       float64(header.CenterLonE7) / 10000000,
-		CenterLat:       float64(header.CenterLatE7) / 10000000,
+		Bounds:          []float64{float64(header.MinLonE7) / 10000000, float64(header.MinLatE7) / 10000000, float64(header.MaxLonE7) / 10000000, float64(header.MaxLatE7) / 10000000},
+		Center:          []float64{float64(header.CenterLonE7) / 10000000, float64(header.CenterLatE7) / 10000000, float64(header.CenterZoom)},
 	}
 }
 

pmtiles/directory_test.go

@@ -102,13 +102,8 @@ func TestHeaderJsonRoundtrip(t *testing.T) {
 	assert.Equal(t, "mvt", j.TileType)
 	assert.Equal(t, 1, j.MinZoom)
 	assert.Equal(t, 3, j.MaxZoom)
-	assert.Equal(t, 2, j.CenterZoom)
-	assert.Equal(t, 1.1, j.MinLon)
-	assert.Equal(t, 2.1, j.MinLat)
-	assert.Equal(t, 1.2, j.MaxLon)
-	assert.Equal(t, 2.2, j.MaxLat)
-	assert.Equal(t, 3.1, j.CenterLon)
-	assert.Equal(t, 3.2, j.CenterLat)
+	assert.Equal(t, []float64{1.1, 2.1, 1.2, 2.2}, j.Bounds)
+	assert.Equal(t, []float64{3.1, 3.2, 2}, j.Center)
 }
 
 func TestOptimizeDirectories(t *testing.T) {
@@ -207,3 +202,30 @@ func TestStringifiedExtension(t *testing.T) {
 	assert.Equal(t, ".webp", headerExt(HeaderV3{TileType: Webp}))
 	assert.Equal(t, ".avif", headerExt(HeaderV3{TileType: Avif}))
 }
+
+func TestStringToTileType(t *testing.T) {
+	assert.Equal(t, "mvt", tileTypeToString(stringToTileType("mvt")))
+	assert.Equal(t, "png", tileTypeToString(stringToTileType("png")))
+	assert.Equal(t, "jpg", tileTypeToString(stringToTileType("jpg")))
+	assert.Equal(t, "webp", tileTypeToString(stringToTileType("webp")))
+	assert.Equal(t, "avif", tileTypeToString(stringToTileType("avif")))
+	assert.Equal(t, "", tileTypeToString(stringToTileType("")))
+}
+
+func TestStringToCompression(t *testing.T) {
+	s, has := compressionToString(stringToCompression("gzip"))
+	assert.True(t, has)
+	assert.Equal(t, "gzip", s)
+	s, has = compressionToString(stringToCompression("br"))
+	assert.True(t, has)
+	assert.Equal(t, "br", s)
+	s, has = compressionToString(stringToCompression("zstd"))
+	assert.True(t, has)
+	assert.Equal(t, "zstd", s)
+	s, has = compressionToString(stringToCompression("none"))
+	assert.False(t, has)
+	assert.Equal(t, "none", s)
+	s, has = compressionToString(stringToCompression("unknown"))
+	assert.False(t, has)
+	assert.Equal(t, "unknown", s)
+}

pmtiles/edit.go

@@ -0,0 +1,152 @@
+package pmtiles
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"github.com/schollz/progressbar/v3"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+)
+
+// Edit parts of the header or metadata.
+// works in-place if only the header is modified.
+func Edit(_ *log.Logger, inputArchive string, newHeaderJSONFile string, newMetadataFile string) error {
+	if newHeaderJSONFile == "" && newMetadataFile == "" {
+		return fmt.Errorf("must supply --header-json and/or --metadata to edit")
+	}
+
+	file, err := os.OpenFile(inputArchive, os.O_RDWR, 0666)
+	defer file.Close()
+	if err != nil {
+		return err
+	}
+
+	buf := make([]byte, 127)
+	_, err = file.Read(buf)
+	if err != nil {
+		return err
+	}
+	oldHeader, err := deserializeHeader(buf)
+	if err != nil {
+		return err
+	}
+
+	newHeader := oldHeader
+
+	if newHeaderJSONFile != "" {
+		newHeaderData := HeaderJson{}
+		data, err := ioutil.ReadFile(newHeaderJSONFile)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(data, &newHeaderData)
+		if err != nil {
+			return err
+		}
+
+		if len(newHeaderData.Bounds) != 4 {
+			return fmt.Errorf("header len(bounds) must == 4")
+		}
+		if len(newHeaderData.Center) != 3 {
+			return fmt.Errorf("header len(center) must == 3")
+		}
+
+		newHeader.TileType = stringToTileType(newHeaderData.TileType)
+		newHeader.TileCompression = stringToCompression(newHeaderData.TileCompression)
+		newHeader.MinZoom = uint8(newHeaderData.MinZoom)
+		newHeader.MaxZoom = uint8(newHeaderData.MaxZoom)
+		newHeader.MinLonE7 = int32(newHeaderData.Bounds[0] * 10000000)
+		newHeader.MinLatE7 = int32(newHeaderData.Bounds[1] * 10000000)
+		newHeader.MaxLonE7 = int32(newHeaderData.Bounds[2] * 10000000)
+		newHeader.MaxLatE7 = int32(newHeaderData.Bounds[3] * 10000000)
+		newHeader.CenterLonE7 = int32(newHeaderData.Center[0] * 10000000)
+		newHeader.CenterLatE7 = int32(newHeaderData.Center[1] * 10000000)
+		newHeader.CenterZoom = uint8(newHeaderData.Center[2])
+	}
+
+	if newMetadataFile == "" {
+		buf = serializeHeader(newHeader)
+		_, err = file.WriteAt(buf, 0)
+		if err != nil {
+			return err
+		}
+		file.Close()
+		return nil
+	}
+
+	newMetadataUncompressed, err := ioutil.ReadFile(newMetadataFile)
+
+	var parsedMetadata map[string]interface{}
+	if err := json.Unmarshal(newMetadataUncompressed, &parsedMetadata); err != nil {
+		return err
+	}
+
+	var metadataBytes bytes.Buffer
+	if oldHeader.InternalCompression != Gzip {
+		return fmt.Errorf("only gzip internal compression is currently supported")
+	}
+
+	w, _ := gzip.NewWriterLevel(&metadataBytes, gzip.BestCompression)
+	w.Write(newMetadataUncompressed)
+	w.Close()
+
+	if err != nil {
+		return err
+	}
+
+	tempFilePath := inputArchive + ".tmp"
+
+	if _, err = os.Stat(tempFilePath); err == nil {
+		return fmt.Errorf("A file with the same name already exists")
+	}
+
+	outfile, err := os.Create(tempFilePath)
+	if err != nil {
+		return err
+	}
+	defer outfile.Close()
+
+	newHeader.MetadataOffset = newHeader.RootOffset + newHeader.RootLength
+	newHeader.MetadataLength = uint64(len(metadataBytes.Bytes()))
+	newHeader.LeafDirectoryOffset = newHeader.MetadataOffset + newHeader.MetadataLength
+	newHeader.TileDataOffset = newHeader.LeafDirectoryOffset + newHeader.LeafDirectoryLength
+
+	bar := progressbar.DefaultBytes(
+		int64(HeaderV3LenBytes+newHeader.RootLength+uint64(len(metadataBytes.Bytes()))+newHeader.LeafDirectoryLength+newHeader.TileDataLength),
+		"writing file",
+	)
+
+	buf = serializeHeader(newHeader)
+	io.Copy(io.MultiWriter(outfile, bar), bytes.NewReader(buf))
+
+	rootSection := io.NewSectionReader(file, int64(oldHeader.RootOffset), int64(oldHeader.RootLength))
+	if _, err := io.Copy(io.MultiWriter(outfile, bar), rootSection); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(io.MultiWriter(outfile, bar), bytes.NewReader(metadataBytes.Bytes())); err != nil {
+		return err
+	}
+
+	leafSection := io.NewSectionReader(file, int64(oldHeader.LeafDirectoryOffset), int64(oldHeader.LeafDirectoryLength))
+	if _, err := io.Copy(io.MultiWriter(outfile, bar), leafSection); err != nil {
+		return err
+	}
+
+	tileSection := io.NewSectionReader(file, int64(oldHeader.TileDataOffset), int64(oldHeader.TileDataLength))
+	if _, err := io.Copy(io.MultiWriter(outfile, bar), tileSection); err != nil {
+		return err
+	}
+
+	// explicitly close in order to rename
+	file.Close()
+	outfile.Close()
+	if err := os.Rename(tempFilePath, inputArchive); err != nil {
+		return err
+	}
+	return nil
+}