update sync

pmtiles/makesync.go

@@ -256,7 +256,6 @@ func Makesync(logger *log.Logger, cli_version string, file string, block_size_kb
 
 func Sync(logger *log.Logger, file string, syncfile string) error {
 	start := time.Now()
-	total_remote_bytes := uint64(0)
 
 	sync, err := os.Open(syncfile)
 	if err != nil {
@@ -305,19 +304,19 @@ func Sync(logger *log.Logger, file string, syncfile string) error {
 		return fmt.Errorf("Error: archive must be clustered for makesync.")
 	}
 
-	// GetHash := func(offset uint64, length uint64) uint64 {
-	// 	hasher := fnv.New64a()
-	// 	r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+offset), int64(length))
-	// 	if err != nil {
-	// 		log.Fatal(err)
-	// 	}
+	GetHash := func(offset uint64, length uint64) uint64 {
+		hasher := fnv.New64a()
+		r, err := bucket.NewRangeReader(ctx, key, int64(header.TileDataOffset+offset), int64(length))
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	// 	if _, err := io.Copy(hasher, r); err != nil {
-	// 		log.Fatal(err)
-	// 	}
-	// 	r.Close()
-	// 	return hasher.Sum64()
-	// }
+		if _, err := io.Copy(hasher, r); err != nil {
+			log.Fatal(err)
+		}
+		r.Close()
+		return hasher.Sum64()
+	}
 
 	var CollectEntries func(uint64, uint64, func(EntryV3))
 
@@ -347,26 +346,47 @@ func Sync(logger *log.Logger, file string, syncfile string) error {
 		"calculating diff",
 	)
 
-	hits := 0
+	wanted := make([]SyncBlock, 0)
+	have := make([]SyncBlock, 0)
+
+	idx := 0
 
 	CollectEntries(header.RootOffset, header.RootLength, func(e EntryV3) {
 		bar.Add(1)
-// 	hash_result := GetHash(e.Offset, potential_match.Length)
-// 	if hash_result == potential_match.Hash {
-// 		hits += 1
-// 		delete(by_start_id, e.TileId)
-// 	}
+
+		if idx < len(blocks) {
+			for e.TileId > blocks[idx].Start {
+				wanted = append(wanted, blocks[idx])
+				idx = idx + 1
+			}
+
+			if e.TileId == blocks[idx].Start {
+				hash_result := GetHash(e.Offset, blocks[idx].Length)
+				if hash_result == blocks[idx].Hash {
+					have = append(have, blocks[idx])
+				} else {
+					wanted = append(wanted, blocks[idx])
+				}
+				idx = idx + 1
+			}
+		}
 	})
 
 	to_transfer := uint64(0)
-	// for _, v := range by_start_id {
-	// 	to_transfer += v.Length
-	// }
+	total_remote_bytes := uint64(0)
+	for _, v := range wanted {
+		to_transfer += v.Length
+		total_remote_bytes += v.Length
+	}
+
+	for _, v := range have {
+		total_remote_bytes += v.Length
+	}
 
-	blocks_matched := float64(hits) / float64(len(blocks)) * 100
+	blocks_matched := float64(len(have)) / float64(len(blocks)) * 100
 	pct := float64(to_transfer) / float64(total_remote_bytes) * 100
 
-	fmt.Printf("%d/%d blocks matched (%.1f%%), need to transfer %s/%s (%.1f%%).\n", hits, len(blocks), blocks_matched, humanize.Bytes(to_transfer), humanize.Bytes(total_remote_bytes), pct)
+	fmt.Printf("%d/%d blocks matched (%.1f%%), need to transfer %s/%s (%.1f%%).\n", len(have), len(blocks), blocks_matched, humanize.Bytes(to_transfer), humanize.Bytes(total_remote_bytes), pct)
 
 	fmt.Printf("Completed sync in %v.\n", time.Since(start))
 	return nil