add --system-tables-only on setup command

CHANGELOG.md

@@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * A change in your SQL schema may be required to keep existing substreams:SQL integrations working:
   * The presence of a primary key (single key or composite) is now *MANDATORY* on every table.
   * The `sf.substreams.sink.database.v1.TableChange` message, generated inside substreams, must now exactly match its primary key with the one in the SQL schema.
-  * You will need to re-run `setup` on your existing PostgreSQL databases to add the `substreams_history` table.
+  * You will need to re-run `setup` on your existing PostgreSQL databases to add the `substreams_history` table. You can use the new `--system-tables-only` flag to perform only that.
 
 * Since reorgs management is not yet supported on Clickhouse, users will have to set `--undo-buffer-size` to a non-zero value (`12` was the previous default)
 

cmd/substreams-sink-sql/setup.go

@@ -19,6 +19,7 @@ var sinkSetupCmd = Command(sinkSetupE,
 	ExactArgs(2),
 	Flags(func(flags *pflag.FlagSet) {
 		flags.Bool("postgraphile", false, "Will append the necessary 'comments' on cursors table to fully support postgraphile")
+		flags.Bool("system-tables-only", false, "will only create/update the systems tables (cursors, substreams_history) and ignore the schema from the manifest")
 		flags.Bool("ignore-duplicate-table-errors", false, "[Dev] Use this if you want to ignore duplicate table errors, take caution that this means the 'schemal.sql' file will not have run fully!")
 	}),
 )
@@ -29,6 +30,7 @@ func sinkSetupE(cmd *cobra.Command, args []string) error {
 	dsn := args[0]
 	manifestPath := args[1]
 	ignoreDuplicateTableErrors := sflags.MustGetBool(cmd, "ignore-duplicate-table-errors")
+	systemTableOnly := sflags.MustGetBool(cmd, "system-tables-only")
 
 	reader, err := manifest.NewReader(manifestPath)
 	if err != nil {
@@ -49,7 +51,12 @@ func sinkSetupE(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("new psql loader: %w", err)
 	}
 
-	err = dbLoader.Setup(ctx, []byte(sinkConfig.Schema), sflags.MustGetBool(cmd, "postgraphile"))
+	schema := sinkConfig.Schema
+	if systemTableOnly {
+		schema = ""
+	}
+
+	err = dbLoader.Setup(ctx, schema, sflags.MustGetBool(cmd, "postgraphile"))
 	if err != nil {
 		if isDuplicateTableError(err) && ignoreDuplicateTableErrors {
 			zlog.Info("received duplicate table error, script dit not executed succesfully completed")

db/db.go

@@ -277,10 +277,11 @@ func (l *Loader) MarshalLogObject(encoder zapcore.ObjectEncoder) error {
 
 // Setup creates the schema, cursors and history table where the <schemaBytes> is a byte array
 // taken from somewhere.
-func (l *Loader) Setup(ctx context.Context, schemaBytes []byte, withPostgraphile bool) error {
-	schemaSql := string(schemaBytes)
-	if err := l.getDialect().ExecuteSetupScript(ctx, l, schemaSql); err != nil {
-		return fmt.Errorf("exec schema: %w", err)
+func (l *Loader) Setup(ctx context.Context, schemaSql string, withPostgraphile bool) error {
+	if schemaSql != "" {
+		if err := l.getDialect().ExecuteSetupScript(ctx, l, schemaSql); err != nil {
+			return fmt.Errorf("exec schema: %w", err)
+		}
 	}
 
 	if err := l.setupCursorTable(ctx, withPostgraphile); err != nil {