algod: Hot/Cold Data Directories and Resource Paths (#5614)

agreement/service.go

@@ -94,10 +94,15 @@ func MakeService(p Parameters) (*Service, error) {
 
 	s.log = makeServiceLogger(p.Logger)
 
+	// If cadaver directory is not set, use cold data directory (which may also not be set)
+	cadaverDir := p.CadaverDirectory
+	if cadaverDir == "" {
+		cadaverDir = p.ColdDataDir
+	}
 	// GOAL2-541: tracer is not concurrency safe. It should only ever be
 	// accessed by main state machine loop.
 	var err error
-	s.tracer, err = makeTracer(s.log, defaultCadaverName, p.CadaverSizeTarget, p.CadaverDirectory,
+	s.tracer, err = makeTracer(s.log, defaultCadaverName, p.CadaverSizeTarget, cadaverDir,
 		s.Local.EnableAgreementReporting, s.Local.EnableAgreementTimeMetrics)
 	if err != nil {
 		return nil, err

catchup/pref_test.go

@@ -61,7 +61,8 @@ func BenchmarkServiceFetchBlocks(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		inMem := true
-		local, err := data.LoadLedger(logging.TestingLog(b), b.Name()+"empty"+strconv.Itoa(i), inMem, protocol.ConsensusCurrentVersion, genesisBalances, "", crypto.Digest{}, nil, cfg)
+		prefix := b.Name() + "empty" + strconv.Itoa(i)
+		local, err := data.LoadLedger(logging.TestingLog(b), prefix, inMem, protocol.ConsensusCurrentVersion, genesisBalances, "", crypto.Digest{}, nil, cfg)
 		require.NoError(b, err)
 
 		// Make Service
@@ -148,7 +149,8 @@ func benchenv(t testing.TB, numAccounts, numBlocks int) (ledger, emptyLedger *da
 	const inMem = true
 	cfg := config.GetDefaultLocal()
 	cfg.Archival = true
-	emptyLedger, err = data.LoadLedger(logging.TestingLog(t), t.Name()+"empty", inMem, protocol.ConsensusCurrentVersion, genesisBalances, "", crypto.Digest{}, nil, cfg)
+	prefix := t.Name() + "empty"
+	emptyLedger, err = data.LoadLedger(logging.TestingLog(t), prefix, inMem, protocol.ConsensusCurrentVersion, genesisBalances, "", crypto.Digest{}, nil, cfg)
 	require.NoError(t, err)
 
 	ledger, err = datatest.FabricateLedger(logging.TestingLog(t), t.Name(), parts, genesisBalances, emptyLedger.LastRound()+basics.Round(numBlocks))

cmd/algod/main.go

@@ -26,8 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gofrs/flock"
-
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/daemon/algod"
@@ -40,6 +38,7 @@ import (
 	"github.com/algorand/go-algorand/util"
 	"github.com/algorand/go-algorand/util/metrics"
 	"github.com/algorand/go-algorand/util/tokens"
+	"github.com/gofrs/flock"
 
 	"github.com/algorand/go-deadlock"
 )
@@ -93,11 +92,13 @@ func run() int {
 	baseHeartbeatEvent.Info.Branch = version.Branch
 	baseHeartbeatEvent.Info.CommitHash = version.GetCommitHash()
 
+	// -b will print only the git branch and then exit
 	if *branchCheck {
 		fmt.Println(config.Branch)
 		return 0
 	}
 
+	// -c will print only the release channel and then exit
 	if *channelCheck {
 		fmt.Println(config.Channel)
 		return 0
@@ -115,24 +116,13 @@ func run() int {
 	}
 
 	genesisPath := *genesisFile
-	if genesisPath == "" {
-		genesisPath = filepath.Join(dataDir, config.GenesisJSONFile)
-	}
-
-	// Load genesis
-	genesisText, err := os.ReadFile(genesisPath)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot read genesis file %s: %v\n", genesisPath, err)
-		return 1
-	}
-
-	var genesis bookkeeping.Genesis
-	err = protocol.DecodeJSON(genesisText, &genesis)
+	genesis, genesisText, err := loadGenesis(dataDir, genesisPath)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot parse genesis file %s: %v\n", genesisPath, err)
+		fmt.Fprintf(os.Stderr, "Error loading genesis file (%s): %v", genesisPath, err)
 		return 1
 	}
 
+	// -G will print only the genesis ID and then exit
 	if *genesisPrint {
 		fmt.Println(genesis.ID())
 		return 0
@@ -453,3 +443,19 @@ func resolveDataDir() string {
 	}
 	return dir
 }
+
+func loadGenesis(dataDir string, genesisPath string) (bookkeeping.Genesis, string, error) {
+	if genesisPath == "" {
+		genesisPath = filepath.Join(dataDir, config.GenesisJSONFile)
+	}
+	genesisText, err := os.ReadFile(genesisPath)
+	if err != nil {
+		return bookkeeping.Genesis{}, "", err
+	}
+	var genesis bookkeeping.Genesis
+	err = protocol.DecodeJSON(genesisText, &genesis)
+	if err != nil {
+		return bookkeeping.Genesis{}, "", err
+	}
+	return genesis, string(genesisText), nil
+}

config/config_test.go

@@ -654,3 +654,152 @@ func TestLocal_RecalculateConnectionLimits(t *testing.T) {
 		})
 	}
 }
+
+// Tests that ensureAbsGenesisDir resolves a path to an absolute path, appends the genesis directory, and creates any needed directories
+func TestEnsureAbsDir(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	testDirectory := t.TempDir()
+
+	t1 := filepath.Join(testDirectory, "test1")
+	t1Abs, err := ensureAbsGenesisDir(t1, "myGenesisID")
+	require.NoError(t, err)
+	require.DirExists(t, t1Abs)
+	require.Equal(t, testDirectory+"/test1/myGenesisID", t1Abs)
+
+	// confirm that relative paths become absolute
+	t2 := filepath.Join(testDirectory, "test2", "..")
+	t2Abs, err := ensureAbsGenesisDir(t2, "myGenesisID")
+	require.NoError(t, err)
+	require.DirExists(t, t2Abs)
+	require.Equal(t, testDirectory+"/myGenesisID", t2Abs)
+}
+
+// TestEnsureAndResolveGenesisDirs confirms that paths provided in the config are resolved to absolute paths and are created if relevant
+func TestEnsureAndResolveGenesisDirs(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+
+	testDirectory := t.TempDir()
+	// insert some "Bad" path elements to see them removed when converted to absolute
+	cfg.TrackerDBDir = filepath.Join(testDirectory, "BAD/../custom_tracker")
+	cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block")
+	cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash")
+	cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof")
+	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
+
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	require.NoError(t, err)
+
+	// confirm that the paths are absolute, and contain the genesisID
+	require.Equal(t, testDirectory+"/custom_tracker/myGenesisID", paths.TrackerGenesisDir)
+	require.DirExists(t, paths.TrackerGenesisDir)
+	require.Equal(t, testDirectory+"/custom_block/myGenesisID", paths.BlockGenesisDir)
+	require.DirExists(t, paths.BlockGenesisDir)
+	require.Equal(t, testDirectory+"/custom_crash/myGenesisID", paths.CrashGenesisDir)
+	require.DirExists(t, paths.CrashGenesisDir)
+	require.Equal(t, testDirectory+"/custom_stateproof/myGenesisID", paths.StateproofGenesisDir)
+	require.DirExists(t, paths.StateproofGenesisDir)
+	require.Equal(t, testDirectory+"/custom_catchpoint/myGenesisID", paths.CatchpointGenesisDir)
+	require.DirExists(t, paths.CatchpointGenesisDir)
+}
+
+// TestEnsureAndResolveGenesisDirs_hierarchy confirms that when only some directories are specified, other directories defer to them
+func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	require.NoError(t, err)
+	// confirm that if only the root is specified, it is used for all directories
+	require.Equal(t, testDirectory+"/myGenesisID", paths.TrackerGenesisDir)
+	require.DirExists(t, paths.TrackerGenesisDir)
+	require.Equal(t, testDirectory+"/myGenesisID", paths.BlockGenesisDir)
+	require.DirExists(t, paths.BlockGenesisDir)
+	require.Equal(t, testDirectory+"/myGenesisID", paths.CrashGenesisDir)
+	require.DirExists(t, paths.CrashGenesisDir)
+	require.Equal(t, testDirectory+"/myGenesisID", paths.StateproofGenesisDir)
+	require.DirExists(t, paths.StateproofGenesisDir)
+	require.Equal(t, testDirectory+"/myGenesisID", paths.CatchpointGenesisDir)
+	require.DirExists(t, paths.CatchpointGenesisDir)
+
+	cfg = GetDefaultLocal()
+	testDirectory = t.TempDir()
+	hot := filepath.Join(testDirectory, "hot")
+	cold := filepath.Join(testDirectory, "cold")
+	cfg.HotDataDir = hot
+	cfg.ColdDataDir = cold
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	require.NoError(t, err)
+	// confirm that if hot/cold are specified, hot/cold are used for appropriate directories
+	require.Equal(t, hot+"/myGenesisID", paths.TrackerGenesisDir)
+	require.DirExists(t, paths.TrackerGenesisDir)
+	require.Equal(t, cold+"/myGenesisID", paths.BlockGenesisDir)
+	require.DirExists(t, paths.BlockGenesisDir)
+	require.Equal(t, cold+"/myGenesisID", paths.CrashGenesisDir)
+	require.DirExists(t, paths.CrashGenesisDir)
+	require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir)
+	require.DirExists(t, paths.StateproofGenesisDir)
+	require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir)
+	require.DirExists(t, paths.CatchpointGenesisDir)
+}
+
+// TestEnsureAndResolveGenesisDirsError confirms that if a path can't be created, an error is returned
+func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+
+	testDirectory := t.TempDir()
+	// insert some "Bad" path elements to see them removed when converted to absolute
+	cfg.TrackerDBDir = filepath.Join(testDirectory, "BAD/../custom_tracker")
+	cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block")
+	cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash")
+	cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof")
+	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
+
+	// first try an error with an empty root dir
+	paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID")
+	require.Empty(t, paths)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "rootDir is required")
+
+	require.NoError(t, os.Chmod(testDirectory, 0200))
+
+	// now try an error with a root dir that can't be written to
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	require.Empty(t, paths)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "permission denied")
+}
+
+// TestResolveLogPaths confirms that log paths are resolved to the most appropriate data directory of the supplied config
+func TestResolveLogPaths(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// on default settings, the log paths should be in the root directory
+	cfg := GetDefaultLocal()
+	log, archive := cfg.ResolveLogPaths("root")
+	require.Equal(t, "root/node.log", log)
+	require.Equal(t, "root/node.archive.log", archive)
+
+	// with supplied hot/cold data directories, they resolve to hot/cold
+	cfg = GetDefaultLocal()
+	cfg.HotDataDir = "hot"
+	cfg.ColdDataDir = "cold"
+	log, archive = cfg.ResolveLogPaths("root")
+	require.Equal(t, "hot/node.log", log)
+	require.Equal(t, "cold/node.archive.log", archive)
+
+	// with supplied hot/cold data AND specific paths directories, they resolve to the specific paths
+	cfg = GetDefaultLocal()
+	cfg.HotDataDir = "hot"
+	cfg.ColdDataDir = "cold"
+	cfg.LogFileDir = "mycoolLogDir"
+	cfg.LogArchiveDir = "myCoolLogArchive"
+	log, archive = cfg.ResolveLogPaths("root")
+	require.Equal(t, "mycoolLogDir/node.log", log)
+	require.Equal(t, "myCoolLogArchive/node.archive.log", archive)
+}