From d53f37b1435b02265bc7275635bcf312de4961c9 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 8 Aug 2024 09:39:52 +0100 Subject: [PATCH] Support local SDK generation (#1405) Parameterization refers to the ability for a provider to vary its schema based on a parameter that is passed to a new `Parameterize` call on the provider interface. The package reference that is returned may then be used to interact with the bespoke schema/packages within. One use case for this kind of provider is Kubernetes _custom resource definitions_ (CRDs), which are custom types that Kubernetes users can define in their clusters. Parameterization enables a world in which the generic published Kubernetes provider can be parameterized with a set of CRD definitions, allowing Pulumi to offer the management of those CRDs as first-class resources. As part of this, users would generate an SDK whose types and functions would allow them to refer to their CRDs just as they would any other Pulumi entity, giving benefits such as static type checking, discovery and code completion, and so on. Today, SDK generation assumes that the package being generated will be published to a language-specific registry (e.g. NPM for NodeJS or PyPi for Python). This caters for the primary use case of e.g. Pulumi offering a set of general-purpose providers whose implementations and backing clouds are publicly available. In the case above however, it's likely that most users' CRD setups will be bespoke and/or private. It would be desirable therefore to be able to generate SDKs suitable for "local" consumption only -- that is, without the need for them to be published to a registry. This commit introduces the `--local` parameter to `bin/pulumi-java-gen` to do just that. When generating a "local" SDK, we omit build files (such as `build.gradle`) and materialize a `version.txt` directly into the generated source tree. This then yields a `src/main/...` tree that can be copied directly into an existing Pulumi program before being used like any other SDK. This commit does not cover this "linking" aspect, nor does it move Java to use the gRPC interface for SDK generation. Both of these are near-term future work that will be completed separately as we iterate upon and enhance the parameterized providers experience. --- CHANGELOG_PENDING.md | 4 +++- pkg/cmd/pulumi-java-gen/command.go | 5 +++++ pkg/cmd/pulumi-java-gen/generate.go | 5 ++++- pkg/codegen/java/gen.go | 32 +++++++++++++++++++++++++---- pkg/codegen/java/gen_test.go | 2 +- pulumi | 2 +- 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 4424a7e3bc3..177f7735eba 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -7,4 +7,6 @@ - Bugfix: Fully qualify `java.lang` types in codegen. -- Add support for parameterized providers. \ No newline at end of file +- Add support for parameterized providers. + +- Add support for local SDK generation. \ No newline at end of file diff --git a/pkg/cmd/pulumi-java-gen/command.go b/pkg/cmd/pulumi-java-gen/command.go index d6515a89c0c..806432f4057 100644 --- a/pkg/cmd/pulumi-java-gen/command.go +++ b/pkg/cmd/pulumi-java-gen/command.go @@ -77,6 +77,7 @@ See https://www.pulumi.com/docs/guides/pulumi-packages/schema/#language-specific var versionArg, javaSdkVersionArg, schemaArg, outArg, overrideArg, buildArg string var overlays []string + var local bool cmd.Flags().StringVar(&versionArg, "version", "", "default semantic version for the generated package") @@ -106,6 +107,9 @@ See https://www.pulumi.com/docs/guides/pulumi-packages/schema/#language-specific cmd.Flags().StringArrayVar(&overlays, "overlay", nil, "path(s) to folders to mix into the generated code by copying") + cmd.Flags().BoolVar(&local, "local", false, + "generate an SDK suitable for local consumption") + cmd.Run = cmdutil.RunFunc(func(_ *cobra.Command, _ []string) error { rootDir, err := os.Getwd() if err != nil { @@ -127,6 +131,7 @@ See https://www.pulumi.com/docs/guides/pulumi-packages/schema/#language-specific RootDir: rootDir, OutputDir: outArg, Overlays: overlays, + Local: local, } if overrideArg != "" { diff --git a/pkg/cmd/pulumi-java-gen/generate.go b/pkg/cmd/pulumi-java-gen/generate.go index 7da55b12b45..15832b9a945 100644 --- a/pkg/cmd/pulumi-java-gen/generate.go +++ b/pkg/cmd/pulumi-java-gen/generate.go @@ -42,6 +42,9 @@ type generateJavaOptions struct { // Optional version to set on the package. Version *semver.Version + + // True if the generator should generate an SDK suitable for local consumption as opposed to a publishable package. + Local bool } func generateJava(cfg generateJavaOptions) error { @@ -77,7 +80,7 @@ func generateJava(cfg generateJavaOptions) error { if err != nil { return err } - files, err := javagen.GeneratePackage("pulumi-java-gen", pkg, extraFiles) + files, err := javagen.GeneratePackage("pulumi-java-gen", pkg, extraFiles, cfg.Local) if err != nil { return err } diff --git a/pkg/codegen/java/gen.go b/pkg/codegen/java/gen.go index 1d31cae8a79..2246f774b92 100644 --- a/pkg/codegen/java/gen.go +++ b/pkg/codegen/java/gen.go @@ -2196,7 +2196,12 @@ func LanguageResources(tool string, pkg *schema.Package) (map[string]LanguageRes return resources, nil } -func GeneratePackage(tool string, pkg *schema.Package, extraFiles map[string][]byte) (map[string][]byte, error) { +func GeneratePackage( + tool string, + pkg *schema.Package, + extraFiles map[string][]byte, + local bool, +) (map[string][]byte, error) { modules, info, err := generateModuleContextMap(tool, pkg) if err != nil { return nil, err @@ -2214,15 +2219,34 @@ func GeneratePackage(tool string, pkg *schema.Package, extraFiles map[string][]b } } - // Finally, emit the build files if requested. + // Currently, packages come bundled with a version.txt resource that is used by generated code to report a version. + // When a build tool is configured, we defer the generation of this file to the build process so that e.g. CI + // processes can set the version to be used when releasing or publishing a package, as opposed to when the code for + // that package is generated. In the case that we are generating a package without a build tool, or a local package + // to be incorporated into a program with an existing build process, we need to emit the version.txt file explicitly + // as part of code generation. + if info.BuildFiles == "" || local { + pkgName := fmt.Sprintf("%s%s", info.BasePackageOrDefault(), pkg.Name) + pkgPath := strings.ReplaceAll(pkgName, ".", "/") + + var version string + if pkg.Version != nil { + version = pkg.Version.String() + } else { + version = "0.0.1" + } + + files.add("src/main/resources/"+pkgPath+"/version.txt", []byte(version)) + return files, nil + } + + // If we are emitting a publishable package with a configured build system, emit those files now. switch info.BuildFiles { case "gradle": if err := genGradleProject(pkg, info, files); err != nil { return nil, err } return files, nil - case "": - return files, nil default: return nil, fmt.Errorf("Only `gradle` value currently supported for the `buildFiles` setting, given `%s`", info.BuildFiles) diff --git a/pkg/codegen/java/gen_test.go b/pkg/codegen/java/gen_test.go index 45222a479dd..3f57d8dd34e 100644 --- a/pkg/codegen/java/gen_test.go +++ b/pkg/codegen/java/gen_test.go @@ -223,7 +223,7 @@ func TestGeneratePackage(t *testing.T) { pkg.Language = map[string]interface{}{ "java": testCase.packageInfo, } - return GeneratePackage(tool, pkg, extraFiles) + return GeneratePackage(tool, pkg, extraFiles, false) }, Language: "java", TestCases: []*test.SDKTest{testCase.sdkTest}, diff --git a/pulumi b/pulumi index 22d28187db0..1416a34e69e 160000 --- a/pulumi +++ b/pulumi @@ -1 +1 @@ -Subproject commit 22d28187db0adb0d25988f92cb83671e2fb2b4e9 +Subproject commit 1416a34e69e6939b7e2ff1415e247a8ac52342de