add rollout configs for version updates

pkg/environment/config.go

@@ -27,6 +27,9 @@ type Config struct {
 	Users struct {
 		Authorized []string `json:"authorized"`
 	} `json:"users"`
+	RolloutUpgrade struct {
+		TestFarms []uint32 `json:"test_farms"`
+	} `json:"rollout_upgrade"`
 }
 
 // Merge, updates current config with cfg merging and override config

pkg/upgrade/README.md

@@ -2,9 +2,11 @@
 
 The upgrade module is responsible to keep a zos node always up to date.
 
-It checks the hub for new releases of zos packages.
+It checks the hub for new releases of zos packages. Also check version and `safe_to_upgrade` flag from the chain.
 
-If a new release is available, it will then update the packages and restart the updated module with the new binaries if required.
+If `safe_to_upgrade` is set to `false` and a new release is available then it will only update the [configured farms](https://github.com/threefoldtech/zos-config/blob/main/config.schema.json#L49)
+
+If `safe_to_upgrade` is set to `true` and a new release is available, it will then update the packages and restart the updated module with the new binaries if required.
 
 ## Usage
 
@@ -27,14 +29,14 @@ Then run the upgrader `upgrader.Run(ctx)`
 
 ## How it works
 
-The upgrader module has two running modes depeding on the booting method.
+The upgrader module has two running modes depending on the booting method.
 
 ### Bootstrap Method
 
 Running the upgrader on a node run with `bootstrap` will periodically check the hub for latest tag,
 and if that tag differs from the current one, it updates the local packages to latest.
 
-If the update failed, the upgrader would attempts to install the packages again every `10 secounds` until all packages are successfully updated to prevent partial updates.
+If the update failed, the upgrader would attempts to install the packages again every `10 seconds` until all packages are successfully updated to prevent partial updates.
 
 The upgrader runs periodically every hour to check for new updates.
 

pkg/upgrade/upgrade.go

@@ -2,6 +2,7 @@ package upgrade
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
@@ -19,7 +20,9 @@ import (
 	"github.com/threefoldtech/0-fs/meta"
 	"github.com/threefoldtech/0-fs/rofs"
 	"github.com/threefoldtech/0-fs/storage"
+	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zos4/pkg/app"
+	"github.com/threefoldtech/zos4/pkg/environment"
 	"github.com/threefoldtech/zos4/pkg/upgrade/hub"
 	"github.com/threefoldtech/zos4/pkg/zinit"
 
@@ -48,6 +51,42 @@ const (
 	ZosPackage = "zos.flist"
 )
 
+type ChainVersion struct {
+	SafeToUpgrade bool   `json:"safe_to_upgrade"`
+	Version       string `json:"version"`
+}
+
+func getRolloutConfig(env environment.Environment) (ChainVersion, []uint32, error) {
+	config, err := environment.GetConfig()
+	if err != nil {
+		return ChainVersion{}, nil, errors.Wrap(err, "failed to get network config")
+	}
+
+	manager := substrate.NewManager(env.SubstrateURL...)
+
+	con, err := manager.Substrate()
+	if err != nil {
+		return ChainVersion{}, nil, err
+	}
+
+	v, err := con.GetZosVersion()
+	if err != nil {
+		return ChainVersion{}, nil, errors.Wrap(err, "failed to get zos version from chain")
+	}
+
+	var chainVersion ChainVersion
+	err = json.Unmarshal([]byte(v), &chainVersion)
+	if err != nil {
+		log.Debug().Err(err).Msg("failed to unmarshal chain version and safe to upgrade flag")
+		chainVersion = ChainVersion{
+			SafeToUpgrade: true,
+			Version:       v,
+		}
+	}
+
+	return chainVersion, config.RolloutUpgrade.TestFarms, nil
+}
+
 // Upgrader is the component that is responsible
 // to keep 0-OS up to date
 type Upgrader struct {
@@ -131,7 +170,7 @@ func NewUpgrader(root string, opts ...UpgraderOption) (*Upgrader, error) {
 	return u, nil
 }
 
-// Run strats the upgrader module and run the update according to the detected boot method
+// Run starts the upgrader module and run the update according to the detected boot method
 func (u *Upgrader) Run(ctx context.Context) error {
 	method := u.boot.DetectBootMethod()
 	if method == BootMethodOther {
@@ -236,6 +275,27 @@ func (u *Upgrader) update() error {
 		return nil
 	}
 
+	env := environment.MustGet()
+	chainVer, testFarms, err := getRolloutConfig(env)
+	if err != nil {
+		return errors.Wrap(err, "failed to get rollout config and version")
+	}
+
+	remoteVer := remote.Target[strings.LastIndex(remote.Target, "/")+1:]
+
+	if env.RunningMode != environment.RunningDev && remoteVer != chainVer.Version {
+		// nothing to do! hub version is not the same as the chain
+		return nil
+	}
+
+	if !chainVer.SafeToUpgrade {
+		if !slices.Contains(testFarms, uint32(env.FarmID)) {
+			// nothing to do! waiting for the flag `safe to upgrade to be enabled after A/B testing`
+			// node is not a part of A/B testing
+			return nil
+		}
+	}
+
 	log.Info().Str("version", filepath.Base(remote.Target)).Msg("updating system...")
 	if err := u.updateTo(remote, &current); err != nil {
 		return errors.Wrapf(err, "failed to update to new tag '%s'", remote.Target)

tools/zos-update-worker/internal/update_worker.go

@@ -104,22 +104,24 @@ func (w *Worker) updateZosVersion(network Network, manager client.Manager) error
 	}
 	defer con.Close()
 
-	currentZosVersions, err := con.GetZosVersion()
+	currentZosVersion, err := con.GetZosVersion()
 	if err != nil {
 		return err
 	}
 
-	type ZosVersions struct {
-		Zos      string
-		ZosLight string
+	type ChainVersion struct {
+		SafeToUpgrade bool   `json:"safe_to_upgrade"`
+		Version       string `json:"version"`
 	}
-	versions := ZosVersions{}
 
-	err = json.Unmarshal([]byte(currentZosVersions), &versions)
+	var chainVersion ChainVersion
+	err = json.Unmarshal([]byte(currentZosVersion), &chainVersion)
 	if err != nil {
-		return err
+		log.Debug().Err(err).Msg("failed to unmarshal chain version")
+		chainVersion.Version = currentZosVersion
 	}
-	log.Debug().Msgf("getting substrate version %v for network %v", versions, network)
+
+	log.Debug().Msgf("getting substrate version %v for network %v", chainVersion.Version, network)
 
 	// now we need to find how dst is relative to src
 	path, err := filepath.Rel(w.dst, w.src)
@@ -128,16 +130,16 @@ func (w *Worker) updateZosVersion(network Network, manager client.Manager) error
 	}
 
 	//zos
-	zosCurrent := fmt.Sprintf("%v/.tag-%v", w.src, versions.Zos)
+	zosCurrent := fmt.Sprintf("%v/.tag-%v", w.src, chainVersion.Version)
 	zosLatest := fmt.Sprintf("%v/%v", w.dst, network)
 	// zos light
-	zosLightCurrent := fmt.Sprintf("%v/.tag-%v", w.src, versions.ZosLight)
+	zosLightCurrent := fmt.Sprintf("%v/.tag-%v", w.src, chainVersion.Version)
 	zosLightLatest := fmt.Sprintf("%v/%v-v4", w.dst, network)
 	// the link is like zosCurrent but it has the path relative from the symlink
 	// point of view (so relative to the symlink, how to reach zosCurrent)
 	// hence the link is instead used in all calls to symlink
-	zosLink := fmt.Sprintf("%v/.tag-%v", path, versions.Zos)
-	zosLightLink := fmt.Sprintf("%v/.tag-%v", path, versions.ZosLight)
+	zosLink := fmt.Sprintf("%v/.tag-%v", path, chainVersion.Version)
+	zosLightLink := fmt.Sprintf("%v/.tag-%v", path, chainVersion.Version)
 
 	// update links for both zos and zoslight
 	if err = w.updateLink(zosCurrent, zosLatest, zosLink); err != nil {
@@ -147,6 +149,11 @@ func (w *Worker) updateZosVersion(network Network, manager client.Manager) error
 }
 
 func (w *Worker) updateLink(current string, latest string, link string) error {
+	// check if current exists
+	if _, err := os.Lstat(current); err != nil {
+		return err
+	}
+
 	// check if symlink exists
 	dst, err := os.Readlink(latest)