From 3af1e46df1edf02d29b115ae7106b0940a347366 Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Mon, 7 Sep 2020 20:20:45 +0200 Subject: [PATCH 1/8] add 'ipfs repo migrate' command this command allows to run the repo migration without starting the daemon. resolves #7471 --- core/commands/commands_test.go | 1 + core/commands/repo.go | 43 +++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 5c076307413..8b1e760f483 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -219,6 +219,7 @@ func TestCommands(t *testing.T) { "/repo/stat", "/repo/verify", "/repo/version", + "/repo/migrate", "/resolve", "/shutdown", "/stats", diff --git a/core/commands/repo.go b/core/commands/repo.go index 307b4ffc58f..7b5b7eacd91 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -11,11 +11,13 @@ import ( "sync" "text/tabwriter" - humanize "github.com/dustin/go-humanize" + oldcmds "github.com/ipfs/go-ipfs/commands" cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" corerepo "github.com/ipfs/go-ipfs/core/corerepo" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + humanize "github.com/dustin/go-humanize" cid "github.com/ipfs/go-cid" bstore "github.com/ipfs/go-ipfs-blockstore" cmds "github.com/ipfs/go-ipfs-cmds" @@ -39,6 +41,7 @@ var RepoCmd = &cmds.Command{ "fsck": repoFsckCmd, "version": repoVersionCmd, "verify": repoVerifyCmd, + "migrate": repoMigrateCmd, }, } @@ -49,8 +52,9 @@ type GcResult struct { } const ( - repoStreamErrorsOptionName = "stream-errors" - repoQuietOptionName = "quiet" + repoStreamErrorsOptionName = "stream-errors" + repoQuietOptionName = "quiet" + repoAllowDowngradeOptionName = "allow-downgrade" ) var repoGcCmd = &cmds.Command{ @@ -375,3 +379,36 @@ var repoVersionCmd = &cmds.Command{ }), }, } + +var repoMigrateCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Apply any outstanding migrations to the repo.", + }, + Options: []cmds.Option{ + cmds.BoolOption(repoAllowDowngradeOptionName, "Allow downgrading to a lower repo version"), + }, + NoRemote: true, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + cctx := env.(*oldcmds.Context) + + _, err := fsrepo.Open(cctx.ConfigRoot) + if err != fsrepo.ErrNeedMigration { + fmt.Println("Repo does not require migration") + return nil + } + + fmt.Println("Found outdated fs-repo, starting migration.") + + err = migrate.RunMigration(fsrepo.RepoVersion) + if err != nil { + fmt.Println("The migrations of fs-repo failed:") + fmt.Printf(" %s\n", err) + fmt.Println("If you think this is a bug, please file an issue and include this whole log output.") + fmt.Println(" https://github.com/ipfs/fs-repo-migrations") + return err + } + + fmt.Println("Repo migrated successfully.") + return nil + }, +} From 395ad25a3d55043aa934e47df735b177988b84fb Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Wed, 9 Sep 2020 12:57:40 +0200 Subject: [PATCH 2/8] add tests for 'ipfs repo migrate' command --- core/commands/repo.go | 2 +- test/sharness/t0066-migration.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 7b5b7eacd91..dba04b3a11d 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -408,7 +408,7 @@ var repoMigrateCmd = &cmds.Command{ return err } - fmt.Println("Repo migrated successfully.") + fmt.Printf("Success: fs-repo has been migrated to version %d.\n", fsrepo.RepoVersion) return nil }, } diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh index aa40fd8a468..bee9ff4c9e6 100755 --- a/test/sharness/t0066-migration.sh +++ b/test/sharness/t0066-migration.sh @@ -84,4 +84,26 @@ test_expect_success "output looks good" ' grep "Please get fs-repo-migrations from https://dist.ipfs.io" daemon_out > /dev/null ' +test_expect_success "ipfs repo migrate succeed" ' + test_expect_code 0 ipfs repo migrate > migrate_out +' + +test_expect_success "output looks good" ' + grep "Found outdated fs-repo, starting migration." migrate_out > /dev/null && + grep "Success: fs-repo has been migrated to version 10" migrate_out > /dev/null +' + +test_expect_success "manually reset repo version to 10" ' + echo "10" > "$IPFS_PATH"/version +' + +test_expect_success "detect repo does not need migration" ' + test_expect_code 0 ipfs repo migrate > migrate_out +' + +test_expect_success "output looks good" ' + grep "Repo does not require migration" migrate_out > /dev/null +' +cat migrate_out + test_done From c480e680554c674ad5558516b9d144203810b29c Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Mon, 14 Sep 2020 08:23:54 +0200 Subject: [PATCH 3/8] add 'allow-downgrade' flag to ipfs repo migrate command --- core/commands/repo.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index dba04b3a11d..c1e930ae26a 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -390,6 +390,7 @@ var repoMigrateCmd = &cmds.Command{ NoRemote: true, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { cctx := env.(*oldcmds.Context) + allowDowngrade, _ := req.Options[repoAllowDowngradeOptionName].(bool) _, err := fsrepo.Open(cctx.ConfigRoot) if err != fsrepo.ErrNeedMigration { @@ -399,7 +400,7 @@ var repoMigrateCmd = &cmds.Command{ fmt.Println("Found outdated fs-repo, starting migration.") - err = migrate.RunMigration(fsrepo.RepoVersion) + err = migrate.RunMigration(fsrepo.RepoVersion, allowDowngrade) if err != nil { fmt.Println("The migrations of fs-repo failed:") fmt.Printf(" %s\n", err) From 47ce267ff6089b04982fb64b246fa51ccb110902 Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Sat, 3 Apr 2021 19:22:00 +0200 Subject: [PATCH 4/8] use new code to start a repo migration --- core/commands/repo.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index c1e930ae26a..7c60ab82c5e 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -400,7 +400,9 @@ var repoMigrateCmd = &cmds.Command{ fmt.Println("Found outdated fs-repo, starting migration.") - err = migrate.RunMigration(fsrepo.RepoVersion, allowDowngrade) + // Fetch migrations from current distribution, or location from environ + fetcher := migrate.NewHttpFetcher(migrate.GetDistPathEnv(migrate.CurrentIpfsDist), "", "go-ipfs", 0) + err = migrate.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", allowDowngrade) if err != nil { fmt.Println("The migrations of fs-repo failed:") fmt.Printf(" %s\n", err) From 73b349dc06c2e48f31e8f647d3559e491e08816f Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Fri, 16 Apr 2021 10:19:00 +0200 Subject: [PATCH 5/8] fix shareness tests --- test/sharness/t0066-migration.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh index bee9ff4c9e6..397e0373809 100755 --- a/test/sharness/t0066-migration.sh +++ b/test/sharness/t0066-migration.sh @@ -90,11 +90,11 @@ test_expect_success "ipfs repo migrate succeed" ' test_expect_success "output looks good" ' grep "Found outdated fs-repo, starting migration." migrate_out > /dev/null && - grep "Success: fs-repo has been migrated to version 10" migrate_out > /dev/null + grep "Success: fs-repo has been migrated to version 11" migrate_out > /dev/null ' -test_expect_success "manually reset repo version to 10" ' - echo "10" > "$IPFS_PATH"/version +test_expect_success "manually reset repo version to 11" ' + echo "11" > "$IPFS_PATH"/version ' test_expect_success "detect repo does not need migration" ' From efd33fb16e458a01231e40580db4dd4b2f899946 Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Sun, 16 May 2021 16:56:26 +0200 Subject: [PATCH 6/8] use $IPFS_REPO_VER variable in tests instead of hardcoded version --- test/sharness/t0066-migration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh index 397e0373809..240c74594a2 100755 --- a/test/sharness/t0066-migration.sh +++ b/test/sharness/t0066-migration.sh @@ -90,11 +90,11 @@ test_expect_success "ipfs repo migrate succeed" ' test_expect_success "output looks good" ' grep "Found outdated fs-repo, starting migration." migrate_out > /dev/null && - grep "Success: fs-repo has been migrated to version 11" migrate_out > /dev/null + grep "Success: fs-repo migrated to version $IPFS_REPO_VER" true_out > /dev/null ' test_expect_success "manually reset repo version to 11" ' - echo "11" > "$IPFS_PATH"/version + echo "$IPFS_REPO_VER" > "$IPFS_PATH"/version ' test_expect_success "detect repo does not need migration" ' From c05a2b474640702ccbfccd2aec365faac0c35022 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 13 Aug 2021 17:08:31 -0700 Subject: [PATCH 7/8] Allow repo migrate command to use IPFS for migration download Additional change to remove leftover debug output from ipfsfetcher.go noticed during testing. --- core/commands/repo.go | 26 ++++++++++++++++++- .../migrations/ipfsfetcher/ipfsfetcher.go | 2 -- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 7c60ab82c5e..9f665731653 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -16,6 +16,7 @@ import ( corerepo "github.com/ipfs/go-ipfs/core/corerepo" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher" humanize "github.com/dustin/go-humanize" cid "github.com/ipfs/go-cid" @@ -400,8 +401,31 @@ var repoMigrateCmd = &cmds.Command{ fmt.Println("Found outdated fs-repo, starting migration.") + // Read Migration section of IPFS config + migrationCfg, err := migrate.ReadMigrationConfig(cctx.ConfigRoot) + if err != nil { + return err + } + + // Define function to create IPFS fetcher. Do not supply an + // already-constructed IPFS fetcher, because this may be expensive and + // not needed according to migration config. Instead, supply a function + // to construct the particular IPFS fetcher implementation used here, + // which is called only if an IPFS fetcher is needed. + newIpfsFetcher := func(distPath string) migrate.Fetcher { + return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot) + } + // Fetch migrations from current distribution, or location from environ - fetcher := migrate.NewHttpFetcher(migrate.GetDistPathEnv(migrate.CurrentIpfsDist), "", "go-ipfs", 0) + fetchDistPath := migrate.GetDistPathEnv(migrate.CurrentIpfsDist) + + // Create fetchers according to migrationCfg.DownloadSources + fetcher, err := migrate.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher) + if err != nil { + return err + } + defer fetcher.Close() + err = migrate.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", allowDowngrade) if err != nil { fmt.Println("The migrations of fs-repo failed:") diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index 88f07b502ee..0d1a0c18d0f 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -239,8 +239,6 @@ func (f *IpfsFetcher) startTempNode(ctx context.Context) error { cancel() // Wait until ipfs is stopped <-node.Context().Done() - - fmt.Println("migration peer", node.Identity, "shutdown") } addrs, err := ipfs.Swarm().LocalAddrs(ctx) From 647cb415c9f63273de6eb615d2e02b17d4046295 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 6 Sep 2021 11:09:17 -0700 Subject: [PATCH 8/8] Increase linter timeout --- mk/golang.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/golang.mk b/mk/golang.mk index 0b2a2c55ae2..52597ab4927 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -70,7 +70,7 @@ test_go_fmt: TEST_GO += test_go_fmt test_go_lint: test/bin/golangci-lint - golangci-lint run ./... + golangci-lint run --timeout 5m ./... .PHONY: test_go_lint test_go: $(TEST_GO)