feat: ref cnt and sharky

ethersphere · Feb 20, 2024 · e30e101 · e30e101
1 parent e5e74a9
commit e30e101
Show file tree

Hide file tree

Showing 8 changed files with 667 additions and 0 deletions.
diff --git a/cmd/bee/cmd/db.go b/cmd/bee/cmd/db.go
@@ -31,6 +31,7 @@ const (
 	optionNameValidationPin  = "validate-pin"
 	optionNameCollectionPin  = "pin"
 	optionNameOutputLocation = "output"
+	optionNameRepair         = "repair"
 )
 
 func (c *command) initDBCmd() {
@@ -172,6 +173,139 @@ func dbCompactCmd(cmd *cobra.Command) {
 	cmd.AddCommand(c)
 }
 
+func dbFixRefCntCmd(cmd *cobra.Command) {
+	c := &cobra.Command{
+		Use:   "fixrefcnt",
+		Short: "Recalculates chunk reference counters.",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			v, err := cmd.Flags().GetString(optionNameVerbosity)
+			if err != nil {
+				return fmt.Errorf("get verbosity: %w", err)
+			}
+			v = strings.ToLower(v)
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %w", err)
+			}
+
+			dataDir, err := cmd.Flags().GetString(optionNameDataDir)
+			if err != nil {
+				return fmt.Errorf("get data-dir: %w", err)
+			}
+			if dataDir == "" {
+				return errors.New("no data-dir provided")
+			}
+
+			repair, err := cmd.Flags().GetBool(optionNameRepair)
+			if err != nil {
+				return fmt.Errorf("get repair: %w", err)
+			}
+
+			logger.Warning("fixrefcnt recalculates the chunk reference counter for all known chunks.")
+			if !repair {
+				logger.Warning("By default, it will only report discrepancies unless --" + optionNameRepair + " is specified.")
+			}
+			logger.Warning("Also, fixrefcnt will not reduce the RefCnt below 1.   This may change in the future.")
+			logger.Warning("    Discrepancies logged at Warning level.")
+			logger.Warning("    Progress logged at Info level.")
+			logger.Warning("    ???? logged at Debug level.")
+
+			if repair {
+				logger.Warning("starting to repair the DB with data-dir", "path", dataDir)
+				logger.Warning("this is VERY experimental and may completely corrupt your localstore!")
+				logger.Warning("you have another 10 seconds to change your mind and kill this process with CTRL-C...")
+				time.Sleep(10 * time.Second)
+				logger.Warning("proceeding with database repair...")
+			}
+
+			localstorePath := path.Join(dataDir, "localstore")
+
+			err = storer.FixRefCnt(context.Background(), localstorePath, &storer.Options{
+				Logger:          logger,
+				RadiusSetter:    noopRadiusSetter{},
+				Batchstore:      new(postage.NoOpBatchStore),
+				ReserveCapacity: node.ReserveCapacity,
+			}, repair)
+			if err != nil {
+				return fmt.Errorf("localstore: %w", err)
+			}
+
+			return nil
+		},
+	}
+	c.Flags().String(optionNameDataDir, "", "data directory")
+	c.Flags().String(optionNameVerbosity, "info", "verbosity level")
+	c.Flags().Bool(optionNameRepair, false, "actually (attempt to) FIX any discovered discrepancies")
+	cmd.AddCommand(c)
+}
+
+func dbFixSharkyCmd(cmd *cobra.Command) {
+	c := &cobra.Command{
+		Use:   "fixsharky",
+		Short: "Removes invalid references to the sharky store.",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			v, err := cmd.Flags().GetString(optionNameVerbosity)
+			if err != nil {
+				return fmt.Errorf("get verbosity: %w", err)
+			}
+			v = strings.ToLower(v)
+			logger, err := newLogger(cmd, v)
+			if err != nil {
+				return fmt.Errorf("new logger: %w", err)
+			}
+
+			dataDir, err := cmd.Flags().GetString(optionNameDataDir)
+			if err != nil {
+				return fmt.Errorf("get data-dir: %w", err)
+			}
+			if dataDir == "" {
+				return errors.New("no data-dir provided")
+			}
+
+			repair, err := cmd.Flags().GetBool(optionNameRepair)
+			if err != nil {
+				return fmt.Errorf("get repair: %w", err)
+			}
+
+			logger.Warning("fixsharky ensures that all references to sharky return a chunk that hashes to the expected reference.")
+			if !repair {
+				logger.Warning("By default, it will only report issues (like validate) unless --" + optionNameRepair + " is specified.")
+			}
+			logger.Warning("Note: This WILL leave (hopefully less impactful) inconsistencies in your localstore.")
+			logger.Warning("    Discrepancies logged at Warning level.")
+			logger.Warning("    Progress logged at Info level.")
+			logger.Warning("    SOC chunks logged at Debug level.")
+
+			if repair {
+				logger.Warning("starting to repair the DB with data-dir", "path", dataDir)
+				logger.Warning("this is VERY experimental and may completely corrupt your localstore!")
+				logger.Warning("you have another 10 seconds to change your mind and kill this process with CTRL-C...")
+				time.Sleep(10 * time.Second)
+				logger.Warning("proceeding with database repair...")
+			}
+
+			localstorePath := path.Join(dataDir, "localstore")
+
+			err = storer.FixSharky(context.Background(), localstorePath, &storer.Options{
+				Logger:          logger,
+				RadiusSetter:    noopRadiusSetter{},
+				Batchstore:      new(postage.NoOpBatchStore),
+				ReserveCapacity: node.ReserveCapacity,
+			}, repair)
+			if err != nil {
+				return fmt.Errorf("localstore: %w", err)
+			}
+
+			return nil
+		},
+	}
+	c.Flags().String(optionNameDataDir, "", "data directory")
+	c.Flags().String(optionNameVerbosity, "info", "verbosity level")
+	c.Flags().Bool(optionNameRepair, false, "actually (attempt to) FIX any discovered inconsistencies")
+	c.Flags().Bool(optionNameValidation, false, "run chunk validation checks again after any repairs")
+	cmd.AddCommand(c)
+}
+
 func dbValidatePinsCmd(cmd *cobra.Command) {
 	c := &cobra.Command{
 		Use:   "validate-pin",