Move migration support to provider-ci

.github/workflows/update-workflows.yml

@@ -84,12 +84,6 @@ jobs:
         if: ${{ inputs.bridged != true }}
         run: |
           cp -r ci-mgmt/native-provider-ci/providers/${{ inputs.provider_name }}/repo/. pulumi-${{ inputs.provider_name }}/.
-      - name: Run source code migrations
-        run: |
-          DIR="$PWD/pulumi-${{ inputs.provider_name }}"
-          cd ci-mgmt/tools/sourcemigrator
-          npm ci
-          npx ts-node index.ts "$DIR"
       - name: Close obsolete PRs started by this workflow
         uses: actions/github-script@v7
         if: ${{ !inputs.skip_closing_prs }}

README.md

@@ -121,14 +121,15 @@ make discovery
 
 ## Automatically editing source code across provider repositories
 
-You can apply ad-hoc source edits across provider repositories even on files that are not managed or generated by ci-mgmt. For example, you might need to automatically update example code to use a newer Pulumi SDK major version dependency or a newer version of the underlying infrastructure such as .NET Framework.  This can be done with sourcemigrator:
+You can apply ad-hoc source edits across provider repositories even on files that are not managed or generated by ci-mgmt. For example, you might need to automatically update example code to use a newer Pulumi SDK major version dependency or a newer version of the underlying infrastructure such as .NET Framework.  This can be done with migrations:
 
-- describe your desired edits as a `SourceMigration` in `tools/sourcemigrator`
+- describe your desired edits as a `SourceMigration` in `provider-ci/internal/pkg/migrations`
 
-- test your changes locally:
+- test your changes locally by running ci-mgmt:
 
-      cd tools/sourcemigrator
-      npx ts-node index.ts ~/code/my-provider
+   ```bash
+   go run . generate -c ../../pulumi-azure/.ci-mgmt.yaml -o ../../pulumi-azure/
+   ```
 
 - stand up a PR to ci-mgmt
 

provider-ci/Makefile

@@ -45,7 +45,7 @@ test-provider/%: PROVIDER_NAME = $*
 test-provider/%: bin/provider-ci $(ACTIONLINT)
 	cd test-providers/$(PROVIDER_NAME) && \
 		find . -type f ! -name '.ci-mgmt.yaml' -delete && \
-		../../bin/provider-ci generate
+		../../bin/provider-ci generate --skip-migrations
 	mkdir -p bin/test-provider
 	rm -rf bin/test-provider/$(PROVIDER_NAME)
 	cp -r test-providers/$(PROVIDER_NAME) bin/test-provider

provider-ci/internal/cmd/generate.go

@@ -12,6 +12,7 @@ type generateArguments struct {
 	OutDir         string
 	TemplateName   string
 	ConfigPath     string
+	SkipMigrations bool
 }
 
 var generateArgs generateArguments
@@ -55,6 +56,7 @@ var generateCmd = &cobra.Command{
 			OutDir:         generateArgs.OutDir,
 			TemplateName:   generateArgs.TemplateName,
 			Config:         config,
+			SkipMigrations: generateArgs.SkipMigrations,
 		})
 		return err
 	},
@@ -67,4 +69,5 @@ func init() {
 	generateCmd.Flags().StringVarP(&generateArgs.OutDir, "out", "o", ".", "directory to write generate files to")
 	generateCmd.Flags().StringVarP(&generateArgs.TemplateName, "template", "t", "", "template name to generate (default \"{config.template}\" or otherwise \"bridged-provider\")")
 	generateCmd.Flags().StringVarP(&generateArgs.ConfigPath, "config", "c", ".ci-mgmt.yaml", "local config file to use")
+	generateCmd.Flags().BoolVar(&generateArgs.SkipMigrations, "skip-migrations", false, "skip running migrations")
 }

provider-ci/internal/pkg/generate.go

@@ -12,6 +12,7 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig"
+	"github.com/pulumi/ci-mgmt/provider-ci/internal/pkg/migrations"
 	"gopkg.in/yaml.v3"
 )
 
@@ -25,6 +26,7 @@ type GenerateOpts struct {
 	OutDir         string
 	TemplateName   string // path inside templates, e.g.: bridged-provider
 	Config         Config // .yaml file containing template config
+	SkipMigrations bool
 }
 
 // Data exposed to text/template that can be referenced in the template code.
@@ -74,6 +76,13 @@ func GeneratePackage(opts GenerateOpts) error {
 			return fmt.Errorf("error rendering template %s: %w", templateDir, err)
 		}
 	}
+	if !opts.SkipMigrations {
+		// Run any relevant migrations
+		err = migrations.Migrate(opts.TemplateName, opts.OutDir)
+		if err != nil {
+			return fmt.Errorf("error running migrations: %w", err)
+		}
+	}
 	return nil
 }
 

provider-ci/internal/pkg/migrations/fixupBridgeImports.go

@@ -0,0 +1,98 @@
+package migrations
+
+import (
+	_ "embed"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+//go:embed fixupBridgeImports.patch
+var fixupBridgeImportsPatch string
+
+type fixupBridgeImports struct{}
+
+func (fixupBridgeImports) Name() string {
+	return "Fixup Bridge Imports"
+}
+func (fixupBridgeImports) ShouldRun(templateName string) bool {
+	return templateName == "bridged-provider"
+}
+func (fixupBridgeImports) Migrate(templateName, outDir string) error {
+	path, cleanup, err := writeTempFile("fixupBridgeImports.patch", fixupBridgeImportsPatch)
+	if err != nil {
+		return fmt.Errorf("error writing patch file: %w", err)
+	}
+	defer cleanup()
+
+	patchCmd := exec.Command("go", "run", "github.com/uber-go/[email protected]", "-p", path, "./provider/resources.go")
+	patchCmd.Stdout = os.Stdout
+	patchCmd.Stderr = os.Stderr
+	patchCmd.Dir = outDir
+	if err = patchCmd.Run(); err != nil {
+		return fmt.Errorf("error running gopatch: %w", err)
+	}
+
+	// Find go.mod files and tidy them
+	goModCmd := exec.Command("find", ".", "-name", "go.mod", "-not", "-path", "./upstream/*")
+	goModCmd.Dir = outDir
+	goModCmd.Stderr = os.Stderr
+	goModOutput, err := goModCmd.Output()
+	if err != nil {
+		return fmt.Errorf("error finding go.mod files: %w\n%s", err, goModOutput)
+	}
+
+	goModPaths := strings.Split(string(goModOutput), "\n")
+	for _, goModPath := range goModPaths {
+		if goModPath == "" {
+			continue
+		}
+		tidyCmd := exec.Command("go", "mod", "tidy")
+		tidyCmd.Dir = filepath.Join(outDir, filepath.Dir(goModPath))
+		tidyCmd.Stdout = os.Stdout
+		tidyCmd.Stderr = os.Stderr
+		err = tidyCmd.Run()
+		if err != nil {
+			return fmt.Errorf("error running go mod tidy: %w", err)
+		}
+	}
+
+	// Find modified .go files and run gofumpt on them
+	gitDiff := exec.Command("git", "diff", "--name-only")
+	gitDiff.Dir = outDir
+	gitDiffOutput, err := gitDiff.Output()
+	if err != nil {
+		return fmt.Errorf("error getting changed files: %w", err)
+	}
+	if len(gitDiffOutput) == 0 {
+		return nil
+	}
+
+	diffLines := strings.Split(string(gitDiffOutput), "\n")
+	modifiedGoFiles := []string{}
+	for _, line := range diffLines {
+		if strings.HasSuffix(line, ".go") {
+			modifiedGoFiles = append(modifiedGoFiles, line)
+		}
+	}
+	if len(modifiedGoFiles) == 0 {
+		return nil
+	}
+
+	for _, file := range modifiedGoFiles {
+		// Tidy each file twice to ensure that the file is formatted correctly
+		for i := 0; i < 2; i++ {
+			gofumptCmd := exec.Command("go", "run", "mvdan.cc/gofumpt@latest", "-w", file)
+			gofumptCmd.Stdout = os.Stdout
+			gofumptCmd.Stderr = os.Stderr
+			gofumptCmd.Dir = outDir
+			err = gofumptCmd.Run()
+			if err != nil {
+				return fmt.Errorf("error running gofumpt: %w", err)
+			}
+		}
+	}
+	return nil
+}

provider-ci/internal/pkg/migrations/migrations.go

@@ -0,0 +1,48 @@
+package migrations
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type Migration interface {
+	Name() string
+	Migrate(templateName, outDir string) error
+	ShouldRun(templateName string) bool
+}
+
+func Migrate(templateName, outDir string) error {
+	migrations := []Migration{
+		fixupBridgeImports{},
+		removeExplicitSDKDependency{},
+	}
+	for i, migration := range migrations {
+		if !migration.ShouldRun(templateName) {
+			fmt.Printf("Migration %d: %s: skipped\n", i+1, migration.Name())
+			continue
+		}
+		fmt.Printf("Migration %d: %s: running\n", i+1, migration.Name())
+		err := migration.Migrate(templateName, outDir)
+		if err != nil {
+			return fmt.Errorf("error running migration %q: %w", migration.Name(), err)
+		}
+	}
+	return nil
+}
+
+// Returns the path to the temporary file and a function to clean it up, or an error.
+func writeTempFile(name, content string) (string, func(), error) {
+	dir, err := os.MkdirTemp(os.TempDir(), "pulumi-provider-ci-migration-files")
+	if err != nil {
+		return "", nil, err
+	}
+	path := filepath.Join(dir, name)
+	f, err := os.Create(path)
+	if err != nil {
+		return "", nil, err
+	}
+	defer f.Close()
+	_, err = f.WriteString(content)
+	return path, func() { os.Remove(f.Name()) }, err
+}

provider-ci/internal/pkg/migrations/removeExplicitSDKDependency.go

@@ -0,0 +1,48 @@
+package migrations
+
+import (
+	_ "embed"
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+//go:embed removeExplicitSDKDependency.patch
+var removeExplicitSDKDependencyPatch string
+
+type removeExplicitSDKDependency struct{}
+
+func (removeExplicitSDKDependency) Name() string {
+	return "remove explicit SDK dependency"
+}
+func (removeExplicitSDKDependency) ShouldRun(templateName string) bool {
+	return templateName == "bridged-provider"
+}
+func (removeExplicitSDKDependency) Migrate(templateName, outDir string) error {
+	path, cleanup, err := writeTempFile("removeExplicitSDKDependency.patch", removeExplicitSDKDependencyPatch)
+	if err != nil {
+		return fmt.Errorf("error writing patch file: %w", err)
+	}
+	defer cleanup()
+
+	patchCmd := exec.Command("go", "run", "github.com/uber-go/[email protected]", "-p", path, "./provider/resources.go")
+	patchCmd.Stdout = os.Stdout
+	patchCmd.Stderr = os.Stderr
+	patchCmd.Dir = outDir
+	if err = patchCmd.Run(); err != nil {
+		return fmt.Errorf("error running gopatch: %w", err)
+	}
+
+	// Tidy twice to ensure that the file is formatted correctly
+	for i := 0; i < 2; i++ {
+		gofumptCmd := exec.Command("go", "run", "mvdan.cc/gofumpt@latest", "-w", "./provider/resources.go")
+		gofumptCmd.Stdout = os.Stdout
+		gofumptCmd.Stderr = os.Stderr
+		gofumptCmd.Dir = outDir
+		err = gofumptCmd.Run()
+		if err != nil {
+			return fmt.Errorf("error running gofumpt: %w", err)
+		}
+	}
+	return nil
+}