From b03d4dec3fb0b9d3f055f72ac7a51ac3882124d7 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 23 Aug 2023 10:30:18 +0200 Subject: [PATCH 1/5] feat: migration from 14 to 15 --- Makefile | 7 +- README.md | 3 +- fs-repo-14-to-15/Makefile | 7 + fs-repo-14-to-15/atomicfile/atomicfile.go | 59 +++++ fs-repo-14-to-15/go.mod | 5 + fs-repo-14-to-15/go.sum | 2 + fs-repo-14-to-15/main.go | 11 + fs-repo-14-to-15/migration/migration.go | 212 ++++++++++++++++++ fs-repo-14-to-15/not-sharness/.gitignore | 1 + .../not-sharness/repotest-golden/config | 20 ++ .../repotest-golden/config.14-to-15.bak | 22 ++ .../not-sharness/repotest-golden/version | 1 + .../not-sharness/repotest-init/config | 22 ++ .../not-sharness/repotest-init/version | 1 + fs-repo-14-to-15/not-sharness/test.sh | 16 ++ .../ipfs/fs-repo-migrations/tools/LICENSE | 21 ++ .../tools/go-migrate/LICENSE | 21 ++ .../tools/go-migrate/README.md | 12 + .../tools/go-migrate/cli.go | 79 +++++++ .../tools/go-migrate/doc.go | 2 + .../tools/go-migrate/migrate.go | 38 ++++ .../fs-repo-migrations/tools/lock/.gitignore | 1 + .../fs-repo-migrations/tools/lock/COPYING | 202 +++++++++++++++++ .../fs-repo-migrations/tools/lock/README.txt | 3 + .../fs-repo-migrations/tools/lock/lock.go | 158 +++++++++++++ .../tools/lock/lock_appengine.go | 32 +++ .../tools/lock/lock_darwin_amd64.go | 80 +++++++ .../tools/lock/lock_freebsd.go | 79 +++++++ .../tools/lock/lock_linux_amd64.go | 80 +++++++ .../tools/lock/lock_linux_arm.go | 81 +++++++ .../tools/lock/lock_plan9.go | 55 +++++ .../tools/lock/lock_sigzero.go | 26 +++ .../fs-repo-migrations/tools/mfsr/mfsr.go | 61 +++++ .../fs-repo-migrations/tools/repolock/lock.go | 40 ++++ .../fs-repo-migrations/tools/stump/LICENSE | 21 ++ .../fs-repo-migrations/tools/stump/README.md | 57 +++++ .../fs-repo-migrations/tools/stump/log.go | 62 +++++ fs-repo-14-to-15/vendor/modules.txt | 7 + fs-repo-migrations/main.go | 2 +- sharness/t0030-simple-migration.sh | 2 +- 40 files changed, 1606 insertions(+), 5 deletions(-) create mode 100644 fs-repo-14-to-15/Makefile create mode 100644 fs-repo-14-to-15/atomicfile/atomicfile.go create mode 100644 fs-repo-14-to-15/go.mod create mode 100644 fs-repo-14-to-15/go.sum create mode 100644 fs-repo-14-to-15/main.go create mode 100644 fs-repo-14-to-15/migration/migration.go create mode 100644 fs-repo-14-to-15/not-sharness/.gitignore create mode 100644 fs-repo-14-to-15/not-sharness/repotest-golden/config create mode 100644 fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak create mode 100644 fs-repo-14-to-15/not-sharness/repotest-golden/version create mode 100644 fs-repo-14-to-15/not-sharness/repotest-init/config create mode 100644 fs-repo-14-to-15/not-sharness/repotest-init/version create mode 100755 fs-repo-14-to-15/not-sharness/test.sh create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/LICENSE create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/LICENSE create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/README.md create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/cli.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/doc.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/migrate.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/.gitignore create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/COPYING create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/README.txt create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_appengine.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_darwin_amd64.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_freebsd.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_amd64.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_arm.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_plan9.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_sigzero.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/mfsr/mfsr.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/repolock/lock.go create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/LICENSE create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/README.md create mode 100644 fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/log.go create mode 100644 fs-repo-14-to-15/vendor/modules.txt diff --git a/Makefile b/Makefile index 09dc4ef5..35a82183 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*) IGNORED_DIRS := $(shell cat ignored-migrations) ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS)) -.PHONY: all build clean cmd sharness test test_go test_12_to_13 test_13_to_14 +.PHONY: all build clean cmd sharness test test_go test_12_to_13 test_13_to_14 test_14_to_15 all: build @@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations: sharness: make -C sharness -test: test_go sharness test_12_to_13 test_13_to_14 +test: test_go sharness test_12_to_13 test_13_to_14 test_14_to_15 clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS)) @make -C sharness clean @@ -49,3 +49,6 @@ test_12_to_13: test_13_to_14: @cd fs-repo-13-to-14/not-sharness && ./test.sh + +test_14_to_15: + @cd fs-repo-14-to-15/not-sharness && ./test.sh diff --git a/README.md b/README.md index 83999fab..7595b653 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Here is the table showing which repo version corresponds to which Kubo version: | 11 | 0.8.0 - 0.11.0 | | 12 | 0.12.0 - 0.17.0 | | 13 | 0.18.0 - 0.20.0 | -| 14 | 0.21.0 - current | +| 14 | 0.21.0 - 0.22.0 | +| 15 | 0.23.0 - current | ### How to Run Migrations diff --git a/fs-repo-14-to-15/Makefile b/fs-repo-14-to-15/Makefile new file mode 100644 index 00000000..3797aca4 --- /dev/null +++ b/fs-repo-14-to-15/Makefile @@ -0,0 +1,7 @@ +.PHONY: build clean + +build: + go build -mod=vendor + +clean: + go clean diff --git a/fs-repo-14-to-15/atomicfile/atomicfile.go b/fs-repo-14-to-15/atomicfile/atomicfile.go new file mode 100644 index 00000000..9294460d --- /dev/null +++ b/fs-repo-14-to-15/atomicfile/atomicfile.go @@ -0,0 +1,59 @@ +// Package atomicfile provides the ability to write a file with an eventual +// rename on Close (using os.Rename). This allows for a file to always be in a +// consistent state and never represent an in-progress write. +// +// NOTE: `os.Rename` may not be atomic on your operating system. +package atomicfile + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// File behaves like os.File, but does an atomic rename operation at Close. +type File struct { + *os.File + path string +} + +// New creates a new temporary file that will replace the file at the given +// path when Closed. +func New(path string, mode os.FileMode) (*File, error) { + f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path)) + if err != nil { + return nil, err + } + if err := os.Chmod(f.Name(), mode); err != nil { + f.Close() + os.Remove(f.Name()) + return nil, err + } + return &File{File: f, path: path}, nil +} + +// Close the file replacing the configured file. +func (f *File) Close() error { + if err := f.File.Close(); err != nil { + os.Remove(f.File.Name()) + return err + } + if err := os.Rename(f.Name(), f.path); err != nil { + return err + } + return nil +} + +// Abort closes the file and removes it instead of replacing the configured +// file. This is useful if after starting to write to the file you decide you +// don't want it anymore. +func (f *File) Abort() error { + if err := f.File.Close(); err != nil { + os.Remove(f.Name()) + return err + } + if err := os.Remove(f.Name()); err != nil { + return err + } + return nil +} diff --git a/fs-repo-14-to-15/go.mod b/fs-repo-14-to-15/go.mod new file mode 100644 index 00000000..74d59055 --- /dev/null +++ b/fs-repo-14-to-15/go.mod @@ -0,0 +1,5 @@ +module github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15 + +go 1.18 + +require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea diff --git a/fs-repo-14-to-15/go.sum b/fs-repo-14-to-15/go.sum new file mode 100644 index 00000000..a2960eed --- /dev/null +++ b/fs-repo-14-to-15/go.sum @@ -0,0 +1,2 @@ +github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc= +github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8= diff --git a/fs-repo-14-to-15/main.go b/fs-repo-14-to-15/main.go new file mode 100644 index 00000000..1e4cce3d --- /dev/null +++ b/fs-repo-14-to-15/main.go @@ -0,0 +1,11 @@ +package main + +import ( + mg14 "github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15/migration" + migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate" +) + +func main() { + m := mg14.Migration{} + migrate.Main(m) +} diff --git a/fs-repo-14-to-15/migration/migration.go b/fs-repo-14-to-15/migration/migration.go new file mode 100644 index 00000000..a3cba62f --- /dev/null +++ b/fs-repo-14-to-15/migration/migration.go @@ -0,0 +1,212 @@ +// package mg14 contains the code to perform 14-15 repository migration in Kubo. +// This handles the following: +// - Replaces bootstrap QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ from /quic to /quic-v1 +// - Removes /quic only addresses from .Addresses.Swarm +package mg14 + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate" + mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr" + lock "github.com/ipfs/fs-repo-migrations/tools/repolock" + log "github.com/ipfs/fs-repo-migrations/tools/stump" + + "github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15/atomicfile" +) + +const backupSuffix = ".14-to-15.bak" + +// Migration implements the migration described above. +type Migration struct{} + +// Versions returns the current version string for this migration. +func (m Migration) Versions() string { + return "14-to-15" +} + +// Reversible returns true, as we keep old config around +func (m Migration) Reversible() bool { + return true +} + +// Apply update the config. +func (m Migration) Apply(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("applying %s repo migration", m.Versions()) + + log.VLog("locking repo at %q", opts.Path) + lk, err := lock.Lock2(opts.Path) + if err != nil { + return err + } + defer lk.Close() + + repo := mfsr.RepoPath(opts.Path) + + log.VLog(" - verifying version is '14'") + if err := repo.CheckVersion("14"); err != nil { + return err + } + + log.Log("> Upgrading config to new format") + + path := filepath.Join(opts.Path, "config") + in, err := os.Open(path) + if err != nil { + return err + } + + // make backup + backup, err := atomicfile.New(path+backupSuffix, 0600) + if err != nil { + return err + } + if _, err := backup.ReadFrom(in); err != nil { + panicOnError(backup.Abort()) + return err + } + if _, err := in.Seek(0, io.SeekStart); err != nil { + panicOnError(backup.Abort()) + return err + } + + // Create a temp file to write the output to on success + out, err := atomicfile.New(path, 0600) + if err != nil { + panicOnError(backup.Abort()) + panicOnError(in.Close()) + return err + } + + if err := convert(in, out); err != nil { + panicOnError(out.Abort()) + panicOnError(backup.Abort()) + panicOnError(in.Close()) + return err + } + + if err := in.Close(); err != nil { + panicOnError(out.Abort()) + panicOnError(backup.Abort()) + } + + if err := repo.WriteVersion("15"); err != nil { + log.Error("failed to update version file to 15") + // There was an error so abort writing the output and clean up temp file + panicOnError(out.Abort()) + panicOnError(backup.Abort()) + return err + } else { + // Write the output and clean up temp file + panicOnError(out.Close()) + panicOnError(backup.Close()) + } + + log.Log("updated version file") + + log.Log("Migration 14 to 15 succeeded") + return nil +} + +// panicOnError is reserved for checks we can't solve transactionally if an error occurs +func panicOnError(e error) { + if e != nil { + panic(fmt.Errorf("error can't be dealt with transactionally: %w", e)) + } +} + +func (m Migration) Revert(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("reverting migration") + lk, err := lock.Lock2(opts.Path) + if err != nil { + return err + } + defer lk.Close() + + repo := mfsr.RepoPath(opts.Path) + if err := repo.CheckVersion("15"); err != nil { + return err + } + + cfg := filepath.Join(opts.Path, "config") + if err := os.Rename(cfg+backupSuffix, cfg); err != nil { + return err + } + + if err := repo.WriteVersion("14"); err != nil { + return err + } + if opts.Verbose { + log.Log("lowered version number to 14") + } + + return nil +} + +// convert converts the config from one version to another +func convert(in io.Reader, out io.Writer) error { + confMap := make(map[string]any) + if err := json.NewDecoder(in).Decode(&confMap); err != nil { + return err + } + + // Upgrade bootstrapper QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ from /quic to /quic-v1 + if b, ok := confMap["Bootstrap"]; ok { + bootstrap, ok := b.([]interface{}) + if !ok { + return fmt.Errorf("invalid type for .Bootstrap got %T expected json array", b) + } + + for i, v := range bootstrap { + if v == "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" { + bootstrap[i] = "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + } + } + } + + // Remove /quic only addresses from .Addresses.Swarm + if a, ok := confMap["Addresses"]; ok { + addresses, ok := a.(map[string]any) + if !ok { + return fmt.Errorf("invalid type for .Addresses got %T expected json map", a) + } + + if s, ok := addresses["Swarm"]; ok { + swarm, ok := s.([]interface{}) + if !ok { + return fmt.Errorf("invalid type for .Addresses.Swarm got %T expected json array", s) + } + + var newSwarm []interface{} + for _, v := range swarm { + if addr, ok := v.(string); ok { + if !strings.Contains(addr, "/quic") || strings.Contains(addr, "/quic-v1") { + newSwarm = append(newSwarm, addr) + } + } else { + newSwarm = append(newSwarm, v) + } + } + addresses["Swarm"] = newSwarm + } + } + + // Save new config + fixed, err := json.MarshalIndent(confMap, "", " ") + if err != nil { + return err + } + + if _, err := out.Write(fixed); err != nil { + return err + } + _, err = out.Write([]byte("\n")) + return err +} diff --git a/fs-repo-14-to-15/not-sharness/.gitignore b/fs-repo-14-to-15/not-sharness/.gitignore new file mode 100644 index 00000000..b749ec06 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/.gitignore @@ -0,0 +1 @@ +repotest diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config b/fs-repo-14-to-15/not-sharness/repotest-golden/config new file mode 100644 index 00000000..36bb5447 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config @@ -0,0 +1,20 @@ +{ + "Addresses": { + "Swarm": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ] + }, + "Bootstrap": [ + "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" + ] +} diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak new file mode 100644 index 00000000..8697aa07 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak @@ -0,0 +1,22 @@ +{ + "Addresses": { + "Swarm": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ] + }, + "Bootstrap": [ + "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" + ] +} diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/version b/fs-repo-14-to-15/not-sharness/repotest-golden/version new file mode 100644 index 00000000..60d3b2f4 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/version @@ -0,0 +1 @@ +15 diff --git a/fs-repo-14-to-15/not-sharness/repotest-init/config b/fs-repo-14-to-15/not-sharness/repotest-init/config new file mode 100644 index 00000000..8697aa07 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/repotest-init/config @@ -0,0 +1,22 @@ +{ + "Addresses": { + "Swarm": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ] + }, + "Bootstrap": [ + "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" + ] +} diff --git a/fs-repo-14-to-15/not-sharness/repotest-init/version b/fs-repo-14-to-15/not-sharness/repotest-init/version new file mode 100644 index 00000000..8351c193 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/repotest-init/version @@ -0,0 +1 @@ +14 diff --git a/fs-repo-14-to-15/not-sharness/test.sh b/fs-repo-14-to-15/not-sharness/test.sh new file mode 100755 index 00000000..7698ef74 --- /dev/null +++ b/fs-repo-14-to-15/not-sharness/test.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -x + +echo "Migration 14 to 15" && +cp -r repotest-init repotest && # init repo +go run .. -verbose -path=repotest && # run forward migration +diff -r repotest-golden repotest && # check forward migration against golden +go run .. -verbose -revert -path=repotest && # run backward migration +diff -r repotest-init repotest # check that after backward migration everything is back to how it used to be + +FINISH="$?" # save exit code + +rm -r repotest # cleanup + +exit "$FINISH" # forward exit code diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/LICENSE b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/LICENSE new file mode 100644 index 00000000..0e323020 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Protocol Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/LICENSE b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/LICENSE new file mode 100644 index 00000000..c7386b3c --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Juan Batiz-Benet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/README.md b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/README.md new file mode 100644 index 00000000..cf96be81 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/README.md @@ -0,0 +1,12 @@ +# go-migrate + +This is a very simple migration framework. See "Migrations" in https://github.com/jbenet/random-ideas/issues/33 + +This package includes: + +- `migrate` package -- lib to write migration programs + +## The model + +The idea here is that we have some thing -- usually a directory -- that needs to be migrated between different representation versions. This may be because there has been an upgrade. + diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/cli.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/cli.go new file mode 100644 index 00000000..2a843597 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/cli.go @@ -0,0 +1,79 @@ +package migrate + +import ( + "flag" + "fmt" + "os" +) + +type Flags struct { + Force bool + Revert bool + Path string // file path to migrate for fs based migrations + Verbose bool + Help bool + NoRevert bool +} + +func SetupFlags() Flags { + f := Flags{} + flag.BoolVar(&f.Force, "f", false, "whether to force a migration (ignores warnings)") + flag.BoolVar(&f.Revert, "revert", false, "whether to apply the migration backwards") + flag.BoolVar(&f.Verbose, "verbose", false, "enable verbose logging") + flag.BoolVar(&f.Help, "help", false, "display help message") + flag.StringVar(&f.Path, "path", "", "file path to migrate for fs based migrations (required)") + flag.BoolVar(&f.NoRevert, "no-revert", false, "do not attempt to automatically revert on failure") + + flag.Parse() + return f +} + +var SupportNoRevert = map[string]bool{ + "4-to-5": true, +} + +func Run(m Migration) error { + f := SetupFlags() + + if f.Help { + flag.Usage() + os.Exit(0) + } + + if f.Path == "" { + flag.Usage() + return fmt.Errorf("missing or empty path; flag '-path ' is required") + } + + if !m.Reversible() { + if f.Revert { + return fmt.Errorf("migration %s is irreversible", m.Versions()) + } + if !f.Force { + return fmt.Errorf("migration %s is irreversible (use -f to proceed)", m.Versions()) + } + } + + if f.NoRevert && !SupportNoRevert[m.Versions()] { + return fmt.Errorf("migration %s does not support the '-no-revert' option", m.Versions()) + } + + if f.Revert { + return m.Revert(Options{ + Flags: f, + Verbose: f.Verbose, + }) + } + + return m.Apply(Options{ + Flags: f, + Verbose: f.Verbose, + }) +} + +func Main(m Migration) { + if err := Run(m); err != nil { + fmt.Fprintf(os.Stderr, "error: %s\n", err) + os.Exit(1) + } +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/doc.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/doc.go new file mode 100644 index 00000000..29e607a1 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/doc.go @@ -0,0 +1,2 @@ +// Package migrate is used to write migrations between representations of things. +package migrate diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/migrate.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/migrate.go new file mode 100644 index 00000000..19d3f93e --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/go-migrate/migrate.go @@ -0,0 +1,38 @@ +package migrate + +import ( + "fmt" +) + +// Options are migration options. For now all flags are options. +type Options struct { + Flags + Verbose bool +} + +// Migration represents +type Migration interface { + + // Versions is the "v-to-v" version string. + Versions() string + + // Reversible returns whether this migration can be reverted. + // Endeavor to make them all reversible. This is here only to warn users + // in case this is not the case. + Reversible() bool + + // Apply applies the migration in question. + Apply(Options) error + + // Revert un-applies the migration in question. This should be best-effort. + // Some migrations are definitively one-way. If so, return an error. + Revert(Options) error +} + +func SplitVersion(s string) (from int, to int) { + _, err := fmt.Scanf(s, "%d-to-%d", &from, &to) + if err != nil { + panic(err.Error()) + } + return +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/.gitignore b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/.gitignore new file mode 100644 index 00000000..b25c15b8 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/COPYING b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/COPYING new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/README.txt b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/README.txt new file mode 100644 index 00000000..a9eeb33d --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/README.txt @@ -0,0 +1,3 @@ +File locking library. + +See http://godoc.org/github.com/camlistore/lock diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock.go new file mode 100644 index 00000000..6268527b --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock.go @@ -0,0 +1,158 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// Lock locks the given file, creating the file if necessary. If the +// file already exists, it must have zero size or an error is returned. +// The lock is an exclusive lock (a write lock), but locked files +// should neither be read from nor written to. Such files should have +// zero size and only exist to co-ordinate ownership across processes. +// +// A nil Closer is returned if an error occurred. Otherwise, close that +// Closer to release the lock. +// +// On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s +// advisory locks. In particular, closing any other file descriptor for the +// same file will release the lock prematurely. +// +// Attempting to lock a file that is already locked by the current process +// has undefined behavior. +// +// On other operating systems, lock will fallback to using the presence and +// content of a file named name + '.lock' to implement locking behavior. +func Lock(name string) (io.Closer, error) { + return lockFn(name) +} + +var lockFn = lockPortable + +// Portable version not using fcntl. Doesn't handle crashes as gracefully, +// since it can leave stale lock files. +// TODO: write pid of owner to lock file and on race see if pid is +// still alive? +func lockPortable(name string) (io.Closer, error) { + absName, err := filepath.Abs(name) + if err != nil { + return nil, fmt.Errorf("can't Lock file %q: can't find abs path: %v", name, err) + } + fi, err := os.Stat(absName) + if err == nil && fi.Size() > 0 { + if isStaleLock(absName) { + os.Remove(absName) + } else { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + } + f, err := os.OpenFile(absName, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("failed to create lock file %s %v", absName, err) + } + if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil { + return nil, err + } + return &lockCloser{f: f, abs: absName}, nil +} + +type pidLockMeta struct { + OwnerPID int +} + +func isStaleLock(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + var meta pidLockMeta + if json.NewDecoder(f).Decode(&meta) != nil { + return false + } + if meta.OwnerPID == 0 { + return false + } + p, err := os.FindProcess(meta.OwnerPID) + if err != nil { + // e.g. on Windows + return true + } + // On unix, os.FindProcess always is true, so we have to send + // it a signal to see if it's alive. + if signalZero != nil { + if p.Signal(signalZero) != nil { + return true + } + } + return false +} + +var signalZero os.Signal // nil or set by lock_sigzero.go + +type lockCloser struct { + f *os.File + abs string + once sync.Once + err error +} + +func (lc *lockCloser) Close() error { + lc.once.Do(lc.close) + return lc.err +} + +func (lc *lockCloser) close() { + if err := lc.f.Close(); err != nil { + lc.err = err + } + if err := os.Remove(lc.abs); err != nil { + lc.err = err + } +} + +var ( + lockmu sync.Mutex + locked = map[string]bool{} // abs path -> true +) + +// unlocker is used by the darwin and linux implementations with fcntl +// advisory locks. +type unlocker struct { + f *os.File + abs string +} + +func (u *unlocker) Close() error { + lockmu.Lock() + // Remove is not necessary but it's nice for us to clean up. + // If we do do this, though, it needs to be before the + // u.f.Close below. + os.Remove(u.abs) + if err := u.f.Close(); err != nil { + return err + } + delete(locked, u.abs) + lockmu.Unlock() + return nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_appengine.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_appengine.go new file mode 100644 index 00000000..ab4cad6a --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_appengine.go @@ -0,0 +1,32 @@ +// +build appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "errors" + "io" +) + +func init() { + lockFn = lockAppEngine +} + +func lockAppEngine(name string) (io.Closer, error) { + return nil, errors.New("Lock not available on App Engine") +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_darwin_amd64.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_darwin_amd64.go new file mode 100644 index 00000000..9fea51fe --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_darwin_amd64.go @@ -0,0 +1,80 @@ +// +build darwin,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + // This type matches C's "struct flock" defined in /usr/include/sys/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start uint64 // sizeof(off_t): 8 + Len uint64 // sizeof(off_t): 8 + Pid uint32 // sizeof(pid_t): 4 + Type uint16 // sizeof(short): 2 + Whence uint16 // sizeof(short): 2 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_freebsd.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_freebsd.go new file mode 100644 index 00000000..d3835d62 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_freebsd.go @@ -0,0 +1,79 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start int64 /* off_t starting offset */ + Len int64 /* off_t len = 0 means until end of file */ + Pid int32 /* pid_t lock owner */ + Type int16 /* short lock type: read/write, etc. */ + Whence int16 /* short type of l_start */ + Sysid int32 /* int remote system id or zero for local */ + }{ + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: int32(os.Getpid()), + Type: syscall.F_WRLCK, + Whence: int16(os.SEEK_SET), + Sysid: 0, + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_amd64.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_amd64.go new file mode 100644 index 00000000..3a7eb00a --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_amd64.go @@ -0,0 +1,80 @@ +// +build linux,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint32 + Whence uint32 + Start uint64 + Len uint64 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint32(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_arm.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_arm.go new file mode 100644 index 00000000..c2a0a102 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_linux_arm.go @@ -0,0 +1,81 @@ +// +build linux,arm +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint16 + Whence uint16 + Start uint32 + Len uint32 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + const F_SETLK = 6 // actual value. syscall package is wrong: golang.org/issue/7059 + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_plan9.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_plan9.go new file mode 100644 index 00000000..bdf4e229 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_plan9.go @@ -0,0 +1,55 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +func init() { + lockFn = lockPlan9 +} + +func lockPlan9(name string) (io.Closer, error) { + var f *os.File + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + return &unlocker{f, abs}, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_sigzero.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_sigzero.go new file mode 100644 index 00000000..fd3ba2db --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/lock/lock_sigzero.go @@ -0,0 +1,26 @@ +// +build !appengine +// +build linux darwin freebsd openbsd netbsd dragonfly + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import "syscall" + +func init() { + signalZero = syscall.Signal(0) +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/mfsr/mfsr.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/mfsr/mfsr.go new file mode 100644 index 00000000..c591f67e --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/mfsr/mfsr.go @@ -0,0 +1,61 @@ +package mfsr + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "strings" +) + +const VersionFile = "version" + +type RepoPath string + +func (rp RepoPath) VersionFile() string { + return path.Join(string(rp), VersionFile) +} + +func (rp RepoPath) Version() (string, error) { + if rp == "" { + return "", fmt.Errorf("invalid repo path \"%s\"", rp) + } + + fn := rp.VersionFile() + if _, err := os.Stat(fn); os.IsNotExist(err) { + return "", VersionFileNotFound(rp) + } + + c, err := ioutil.ReadFile(fn) + if err != nil { + return "", err + } + + s := string(c) + s = strings.TrimSpace(s) + return s, nil +} + +func (rp RepoPath) CheckVersion(version string) error { + v, err := rp.Version() + if err != nil { + return err + } + + if v != version { + return fmt.Errorf("versions differ (expected: %s, actual:%s)", version, v) + } + + return nil +} + +func (rp RepoPath) WriteVersion(version string) error { + fn := rp.VersionFile() + return ioutil.WriteFile(fn, []byte(version+"\n"), 0644) +} + +type VersionFileNotFound string + +func (v VersionFileNotFound) Error() string { + return "no version file in repo at " + string(v) +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/repolock/lock.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/repolock/lock.go new file mode 100644 index 00000000..41a19a0f --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/repolock/lock.go @@ -0,0 +1,40 @@ +package lock + +import ( + "fmt" + "io" + "os" + "path" + + "github.com/ipfs/fs-repo-migrations/tools/lock" +) + +var errRepoLock = `failed to acquire repo lock at %s/%s +Is a daemon running? please stop it before running migration` + +// LockFile is the filename of the daemon lock, relative to config dir +// lock changed names. +const ( + LockFile1 = "daemon.lock" + LockFile2 = "repo.lock" +) + +func Lock1(confdir string) (io.Closer, error) { + c, err := lock.Lock(path.Join(confdir, LockFile1)) + if err != nil { + return nil, fmt.Errorf(errRepoLock, confdir, LockFile1) + } + return c, nil +} + +func Remove1(confdir string) error { + return os.Remove(path.Join(confdir, LockFile1)) +} + +func Lock2(confdir string) (io.Closer, error) { + c, err := lock.Lock(path.Join(confdir, LockFile2)) + if err != nil { + return nil, fmt.Errorf(errRepoLock, confdir, LockFile2) + } + return c, nil +} diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/LICENSE b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/LICENSE new file mode 100644 index 00000000..1a976e72 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Jeromy Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/README.md b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/README.md new file mode 100644 index 00000000..ea469730 --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/README.md @@ -0,0 +1,57 @@ +# Stump +A simple log library, for when you don't really care to have super fancy logs. + +Stump has four main log functions, `Log`, `VLog`, `Error` and `Fatal`. + +`Log` is a basic log that always is shown. + +`VLog` is only shown when `stump.Verbose` is set to true. + +`Error` prints a prefix of `ERROR: ` before your log message, +the prefix is configurable by setting `stump.ErrorPrefix`. + +`Fatal` is an error log that also calls `os.Exit` right afterwards. + +## Installation +``` +$ go get -u github.com/whyrusleeping/stump +``` + +## Usage + +```go +import "github.com/whyrusleeping/stump" + +func main() { + stump.Log("Hello World!") + + name := GetName() + stump.Log("My name is %s, do you like it?", name) + + err := DoThing() + if err != nil { + stump.Error(err) + // or + stump.Error("Got an error doing thing: ", err) + // or + stump.Error("Got error '%s' doing thing.", err) + } + + err = DoImportantThing() + if err != nil { + Fatal(err) + } +} +``` + +## Tips +While generally frowned upon, I like importing stump into my packages namespace like so: +``` +import . "github.com/whyrusleeping/stump" +``` + +This allows you to call all the logging functions without the package prefix. +(eg. just `Log("hello")` instead of `stump.Log("hello")`) + +## License +MIT diff --git a/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/log.go b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/log.go new file mode 100644 index 00000000..8b90f90a --- /dev/null +++ b/fs-repo-14-to-15/vendor/github.com/ipfs/fs-repo-migrations/tools/stump/log.go @@ -0,0 +1,62 @@ +package stump + +import ( + "fmt" + "io" + "os" + "strings" +) + +var Verbose bool + +var ErrorPrefix = "ERROR: " + +var LogOut io.Writer = os.Stdout +var ErrOut io.Writer = os.Stdout + +func Error(args ...interface{}) { + log(ErrOut, ErrorPrefix, args) +} + +func Fatal(args ...interface{}) { + Error(args...) + os.Exit(1) +} + +func Log(args ...interface{}) { + log(LogOut, "", args) +} + +func VLog(args ...interface{}) { + if Verbose { + log(LogOut, "", args) + } +} + +func log(out io.Writer, prefix string, args []interface{}) { + writelog := func(format string, args ...interface{}) { + n := strings.Count(format, "%") + if n < len(args) { + format += strings.Repeat(" %s", len(args)-n) + } + if !strings.HasSuffix(format, "\n") { + format += "\n" + } + fmt.Fprintf(out, format, args...) + } + + if len(args) == 0 { + writelog(prefix) + return + } + + switch s := args[0].(type) { + case string: + writelog(prefix+s, args[1:]...) + case fmt.Stringer: + writelog(prefix+s.String(), args[1:]...) + default: + format := strings.Repeat("%s ", len(args)) + writelog(prefix+format, args...) + } +} diff --git a/fs-repo-14-to-15/vendor/modules.txt b/fs-repo-14-to-15/vendor/modules.txt new file mode 100644 index 00000000..d5ef54e8 --- /dev/null +++ b/fs-repo-14-to-15/vendor/modules.txt @@ -0,0 +1,7 @@ +# github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea +## explicit; go 1.14 +github.com/ipfs/fs-repo-migrations/tools/go-migrate +github.com/ipfs/fs-repo-migrations/tools/lock +github.com/ipfs/fs-repo-migrations/tools/mfsr +github.com/ipfs/fs-repo-migrations/tools/repolock +github.com/ipfs/fs-repo-migrations/tools/stump diff --git a/fs-repo-migrations/main.go b/fs-repo-migrations/main.go index 2087588b..903d07e4 100644 --- a/fs-repo-migrations/main.go +++ b/fs-repo-migrations/main.go @@ -130,7 +130,7 @@ func latestRepoMigration(fetcher migrations.Fetcher) (int, error) { // request and calculate latest // // When searching for latest migration, start looking using this repo version - const currentVersion = 13 + const currentVersion = 14 ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() diff --git a/sharness/t0030-simple-migration.sh b/sharness/t0030-simple-migration.sh index 2c3679d1..c2455003 100755 --- a/sharness/t0030-simple-migration.sh +++ b/sharness/t0030-simple-migration.sh @@ -4,7 +4,7 @@ test_description="Simple fs-repo-migrations tests" . lib/test-lib.sh -latestRepoVersion="13" +latestRepoVersion="14" test_expect_success "fs-repo-migrations binary is here" ' test -f "$LOCAL_FS_REPO_MIG" From 5d24f2e5cdf611dbd19760c3c40b2ebbdb7e5ecc Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 23 Aug 2023 10:57:34 +0200 Subject: [PATCH 2/5] chore: update to go 1.20 Co-authored-by: Jorropo --- fs-repo-14-to-15/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs-repo-14-to-15/go.mod b/fs-repo-14-to-15/go.mod index 74d59055..4154d172 100644 --- a/fs-repo-14-to-15/go.mod +++ b/fs-repo-14-to-15/go.mod @@ -1,5 +1,5 @@ module github.com/ipfs/fs-repo-migrations/fs-repo-14-to-15 -go 1.18 +go 1.20 require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea From 4dbea75a951053f4c38831bf6682ec0aef47045a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 28 Aug 2023 23:16:18 +0200 Subject: [PATCH 3/5] feat: cleanup Gateway.HTTPHeaders Closes https://github.com/ipfs/kubo/issues/10005 --- fs-repo-14-to-15/migration/migration.go | 36 +++++++++++++++++++ .../not-sharness/repotest-golden/config | 5 ++- .../repotest-golden/config.14-to-15.bak | 17 ++++++++- .../not-sharness/repotest-init/config | 17 ++++++++- ignored-migrations | 1 + 5 files changed, 73 insertions(+), 3 deletions(-) diff --git a/fs-repo-14-to-15/migration/migration.go b/fs-repo-14-to-15/migration/migration.go index a3cba62f..8b531788 100644 --- a/fs-repo-14-to-15/migration/migration.go +++ b/fs-repo-14-to-15/migration/migration.go @@ -198,6 +198,42 @@ func convert(in io.Reader, out io.Writer) error { } } + // Remove legacy Gateway.HTTPHeaders values that were hardcoded since years ago, but no longer necessary + // (but leave as-is if user made any changes) + // https://github.com/ipfs/kubo/issues/10005 + if a, ok := confMap["Gateway"]; ok { + addresses, ok := a.(map[string]any) + if !ok { + return fmt.Errorf("invalid type for .Gateway got %T expected json map", a) + } + + if s, ok := addresses["HTTPHeaders"]; ok { + headers, ok := s.(map[string]any) + if !ok { + return fmt.Errorf("invalid type for .Gateway.HTTPHeaders got %T expected json map", s) + } + + if acaos, ok := headers["Access-Control-Allow-Origin"].([]interface{}); ok && len(acaos) == 1 && acaos[0] == "*" { + delete(headers, "Access-Control-Allow-Origin") + } else { + return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Origin] got %T expected json array", headers["Access-Control-Allow-Origin"]) + } + + if acams, ok := headers["Access-Control-Allow-Methods"].([]interface{}); ok && len(acams) == 1 && acams[0] == "GET" { + delete(headers, "Access-Control-Allow-Methods") + } else { + return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Methods] got %T expected json array", headers["Access-Control-Allow-Methods"]) + } + if acahs, ok := headers["Access-Control-Allow-Headers"].([]interface{}); ok && len(acahs) == 3 { + if acahs[0] == "X-Requested-With" && acahs[1] == "Range" && acahs[2] == "User-Agent" { + delete(headers, "Access-Control-Allow-Headers") + } + } else { + return fmt.Errorf("invalid type for .Gateway.HTTPHeaders[Access-Control-Allow-Headers] got %T expected json array", headers["Access-Control-Allow-Headers"]) + } + } + } + // Save new config fixed, err := json.MarshalIndent(confMap, "", " ") if err != nil { diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config b/fs-repo-14-to-15/not-sharness/repotest-golden/config index 36bb5447..0488c0ce 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-golden/config +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config @@ -16,5 +16,8 @@ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" - ] + ], + "Gateway": { + "HTTPHeaders": {} + } } diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak index 8697aa07..799b8f99 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak @@ -18,5 +18,20 @@ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" - ] + ], + "Gateway": { + "HTTPHeaders": { + "Access-Control-Allow-Headers": [ + "X-Requested-With", + "Range", + "User-Agent" + ], + "Access-Control-Allow-Methods": [ + "GET" + ], + "Access-Control-Allow-Origin": [ + "*" + ] + } + } } diff --git a/fs-repo-14-to-15/not-sharness/repotest-init/config b/fs-repo-14-to-15/not-sharness/repotest-init/config index 8697aa07..799b8f99 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-init/config +++ b/fs-repo-14-to-15/not-sharness/repotest-init/config @@ -18,5 +18,20 @@ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" - ] + ], + "Gateway": { + "HTTPHeaders": { + "Access-Control-Allow-Headers": [ + "X-Requested-With", + "Range", + "User-Agent" + ], + "Access-Control-Allow-Methods": [ + "GET" + ], + "Access-Control-Allow-Origin": [ + "*" + ] + } + } } diff --git a/ignored-migrations b/ignored-migrations index 0ddb5807..dbc571e1 100644 --- a/ignored-migrations +++ b/ignored-migrations @@ -10,3 +10,4 @@ fs-repo-8-to-9 fs-repo-9-to-10 fs-repo-10-to-11 fs-repo-11-to-12 +fs-repo-12-to-13 From 15439f1a4ec07efafa8ca259a2e01296950871e3 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 28 Aug 2023 23:23:00 +0200 Subject: [PATCH 4/5] chore: skip <13-to-14 in tests and build been a while, and it was pre go 1.20, skipping saves time and avoids potential issues when we build --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 35a82183..290b1182 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*) IGNORED_DIRS := $(shell cat ignored-migrations) ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS)) -.PHONY: all build clean cmd sharness test test_go test_12_to_13 test_13_to_14 test_14_to_15 +.PHONY: all build clean cmd sharness test test_go test_13_to_14 test_14_to_15 all: build @@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations: sharness: make -C sharness -test: test_go sharness test_12_to_13 test_13_to_14 test_14_to_15 +test: test_go sharness test_13_to_14 test_14_to_15 clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS)) @make -C sharness clean From 00a9f01fb6c33b542250e08ebf732db0bf2fc7b7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 31 Aug 2023 16:31:05 +0200 Subject: [PATCH 5/5] 14-to-15: add filtering to All local addresses fields --- fs-repo-14-to-15/migration/migration.go | 11 +++++-- .../not-sharness/repotest-golden/config | 24 +++++++++++++++ .../repotest-golden/config.14-to-15.bak | 30 +++++++++++++++++++ .../not-sharness/repotest-init/config | 30 +++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/fs-repo-14-to-15/migration/migration.go b/fs-repo-14-to-15/migration/migration.go index 8b531788..7f653259 100644 --- a/fs-repo-14-to-15/migration/migration.go +++ b/fs-repo-14-to-15/migration/migration.go @@ -178,10 +178,15 @@ func convert(in io.Reader, out io.Writer) error { return fmt.Errorf("invalid type for .Addresses got %T expected json map", a) } - if s, ok := addresses["Swarm"]; ok { + for _, addressToRemove := range [...]string{"Swarm", "Announce", "AppendAnnounce", "NoAnnounce"} { + s, ok := addresses[addressToRemove] + if !ok { + continue + } + swarm, ok := s.([]interface{}) if !ok { - return fmt.Errorf("invalid type for .Addresses.Swarm got %T expected json array", s) + return fmt.Errorf("invalid type for .Addresses.%s got %T expected json array", addressToRemove, s) } var newSwarm []interface{} @@ -194,7 +199,7 @@ func convert(in io.Reader, out io.Writer) error { newSwarm = append(newSwarm, v) } } - addresses["Swarm"] = newSwarm + addresses[addressToRemove] = newSwarm } } diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config b/fs-repo-14-to-15/not-sharness/repotest-golden/config index 0488c0ce..0593832f 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-golden/config +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config @@ -1,5 +1,29 @@ { "Addresses": { + "Announce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "AppendAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "NoAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], "Swarm": [ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001", diff --git a/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak index 799b8f99..057e7a98 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak +++ b/fs-repo-14-to-15/not-sharness/repotest-golden/config.14-to-15.bak @@ -1,5 +1,35 @@ { "Addresses": { + "Announce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "AppendAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "NoAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], "Swarm": [ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001", diff --git a/fs-repo-14-to-15/not-sharness/repotest-init/config b/fs-repo-14-to-15/not-sharness/repotest-init/config index 799b8f99..057e7a98 100644 --- a/fs-repo-14-to-15/not-sharness/repotest-init/config +++ b/fs-repo-14-to-15/not-sharness/repotest-init/config @@ -1,5 +1,35 @@ { "Addresses": { + "Announce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "AppendAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], + "NoAnnounce": [ + "/ip4/0.0.0.0/tcp/4001", + "/ip6/::/tcp/4001", + "/ip4/0.0.0.0/udp/4001/quic", + "/ip4/0.0.0.0/udp/4001/quic-v1", + "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport", + "/ip6/::/udp/4001/quic", + "/ip6/::/udp/4001/quic-v1", + "/ip6/::/udp/4001/quic-v1/webtransport" + ], "Swarm": [ "/ip4/0.0.0.0/tcp/4001", "/ip6/::/tcp/4001",