diff --git a/api/golang/core/lib/enclaves/kurtosis_yaml.go b/api/golang/core/lib/enclaves/kurtosis_yaml.go index 0473b18238..dce74a9152 100644 --- a/api/golang/core/lib/enclaves/kurtosis_yaml.go +++ b/api/golang/core/lib/enclaves/kurtosis_yaml.go @@ -17,6 +17,10 @@ type KurtosisYaml struct { PackageReplaceOptions map[string]string `yaml:"replace"` } +func NewKurtosisYaml(packageName string, packageDescription string, packageReplaceOptions map[string]string) *KurtosisYaml { + return &KurtosisYaml{PackageName: packageName, PackageDescription: packageDescription, PackageReplaceOptions: packageReplaceOptions} +} + func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) { kurtosisYamlContents, err := os.ReadFile(kurtosisYamlFilepath) if err != nil { diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url.go b/api/golang/core/lib/shared_utils/parsed_git_url.go similarity index 68% rename from core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url.go rename to api/golang/core/lib/shared_utils/parsed_git_url.go index 89e006079e..5a413712b8 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url.go +++ b/api/golang/core/lib/shared_utils/parsed_git_url.go @@ -1,20 +1,20 @@ -package git_package_content_provider +package shared_utils import ( "fmt" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/stacktrace" "net/url" "path" "strings" ) const ( - httpsSchema = "https" - urlPathSeparator = "/" - // for a valid GitURl we need it to look like github.com/author/moduleName + GithubDomainPrefix = "github.com" + httpsSchema = "https" + UrlPathSeparator = "/" + //MinimumSubPathsForValidGitURL for a valid GitURl we need it to look like github.com/author/moduleName // the last two are the minimum requirements for a valid Startosis URL - minimumSubPathsForValidGitURL = 2 + MinimumSubPathsForValidGitURL = 2 tagBranchOrCommitDelimiter = "@" emptyTagBranchOrCommit = "" @@ -54,44 +54,75 @@ func newParsedGitURL(moduleAuthor, moduleName, gitURL, relativeRepoPath, relativ } } -// parseGitURL this takes a Git url (GitHub) for now and converts it into the struct ParsedGitURL +func (parsedUrl *ParsedGitURL) GetModuleAuthor() string { + return parsedUrl.moduleAuthor +} + +func (parsedUrl *ParsedGitURL) GetModuleName() string { + return parsedUrl.moduleName +} + +func (parsedUrl *ParsedGitURL) GetGitURL() string { + return parsedUrl.gitURL +} + +func (parsedUrl *ParsedGitURL) GetRelativeRepoPath() string { + return parsedUrl.relativeRepoPath +} + +func (parsedUrl *ParsedGitURL) GetRelativeFilePath() string { + return parsedUrl.relativeFilePath +} + +func (parsedUrl *ParsedGitURL) GetTagBranchOrCommit() string { + return parsedUrl.tagBranchOrCommit +} + +func (parsedUrl *ParsedGitURL) GetAbsoluteLocatorRelativeToThisURL(relativeUrl string) string { + if strings.HasPrefix(relativeUrl, packageRootPrefixIndicatorInRelativeLocators) { + return path.Join(GithubDomainPrefix, parsedUrl.relativeRepoPath, relativeUrl) + } + return path.Join(GithubDomainPrefix, path.Dir(parsedUrl.relativeFilePath), relativeUrl) +} + +// ParseGitURL this takes a Git url (GitHub) for now and converts it into the struct ParsedGitURL // This can in the future be extended to GitLab or BitBucket or any other Git Host -func parseGitURL(packageURL string) (*ParsedGitURL, *startosis_errors.InterpretationError) { +func ParseGitURL(packageURL string) (*ParsedGitURL, error) { // we expect something like github.com/author/module/path.star // we don't want schemas parsedURL, err := url.Parse(packageURL) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Error parsing the URL of module '%v'", packageURL) + return nil, stacktrace.Propagate(err, "Error parsing the URL of module '%v'", packageURL) } if parsedURL.Scheme != "" { - return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module '%v'. Expected schema to be empty got '%v'", packageURL, parsedURL.Scheme) + return nil, stacktrace.NewError("Error parsing the URL of module '%v'. Expected schema to be empty got '%v'", packageURL, parsedURL.Scheme) } // we prefix schema and make sure that the URL still parses packageURLPrefixedWithHttps := httpsSchema + "://" + packageURL parsedURL, err = url.Parse(packageURLPrefixedWithHttps) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Error parsing the URL with scheme for module '%v'", packageURLPrefixedWithHttps) + return nil, stacktrace.Propagate(err, "Error parsing the URL with scheme for module '%v'", packageURLPrefixedWithHttps) } - if parsedURL.Host != startosis_constants.GithubDomainPrefix { - return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module. We only support modules on Github for now but got '%v'", packageURL) + if parsedURL.Host != GithubDomainPrefix { + return nil, stacktrace.NewError("Error parsing the URL of module. We only support modules on Github for now but got '%v'", packageURL) } pathWithoutVersion, maybeTagBranchOrCommit := parseOutTagBranchOrCommit(parsedURL.Path) splitURLPath := cleanPathAndSplit(pathWithoutVersion) - if len(splitURLPath) < minimumSubPathsForValidGitURL { - return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module: '%v'. The path should contain at least %d subpaths got '%v'", packageURL, minimumSubPathsForValidGitURL, splitURLPath) + if len(splitURLPath) < MinimumSubPathsForValidGitURL { + return nil, stacktrace.NewError("Error parsing the URL of module: '%v'. The path should contain at least %d subpaths got '%v'", packageURL, MinimumSubPathsForValidGitURL, splitURLPath) } moduleAuthor := splitURLPath[0] moduleName := splitURLPath[1] - gitURL := fmt.Sprintf("%v://%v/%v/%v.git", httpsSchema, startosis_constants.GithubDomainPrefix, moduleAuthor, moduleName) + gitURL := fmt.Sprintf("%v://%v/%v/%v.git", httpsSchema, GithubDomainPrefix, moduleAuthor, moduleName) relativeModulePath := path.Join(moduleAuthor, moduleName) relativeFilePath := "" - if len(splitURLPath) > minimumSubPathsForValidGitURL { + if len(splitURLPath) > MinimumSubPathsForValidGitURL { relativeFilePath = path.Join(splitURLPath...) } @@ -107,17 +138,10 @@ func parseGitURL(packageURL string) (*ParsedGitURL, *startosis_errors.Interpreta return parsedGitURL, nil } -func (parsedUrl *ParsedGitURL) getAbsoluteLocatorRelativeToThisURL(relativeUrl string) string { - if strings.HasPrefix(relativeUrl, packageRootPrefixIndicatorInRelativeLocators) { - return path.Join(startosis_constants.GithubDomainPrefix, parsedUrl.relativeRepoPath, relativeUrl) - } - return path.Join(startosis_constants.GithubDomainPrefix, path.Dir(parsedUrl.relativeFilePath), relativeUrl) -} - // cleanPath removes empty "" from the string slice func cleanPathAndSplit(urlPath string) []string { cleanPath := path.Clean(urlPath) - splitPath := strings.Split(cleanPath, urlPathSeparator) + splitPath := strings.Split(cleanPath, UrlPathSeparator) var sliceWithoutEmptyStrings []string for _, subPath := range splitPath { if subPath != "" { @@ -138,7 +162,7 @@ func parseOutTagBranchOrCommit(input string) (string, string) { // 3- github.com/kurtosis-tech/sample-dependency-package/main.star@foo/bar - here the tag is foo/bar; // 4- github.com/kurtosis-tech/sample-dependency-package@foo/bar/mains.tar - here the tag is foo/bar; while file is /kurtosis-tech/sample-dependency-package/main.star // we check if there is a file in maybeTagBranchOrCommitWithFile and then add it to pathWithoutVersion - maybeTagBranchOrCommit, lastSectionOfTagBranchCommitWithFile, _ := cutLast(maybeTagBranchOrCommitWithFile, urlPathSeparator) + maybeTagBranchOrCommit, lastSectionOfTagBranchCommitWithFile, _ := cutLast(maybeTagBranchOrCommitWithFile, UrlPathSeparator) if lastSectionOfTagBranchCommitWithFile != "" && strings.Contains(lastSectionOfTagBranchCommitWithFile, extensionCharacter) { // we assume pathWithoutVersion does not contain a file inside yet diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url_test.go b/api/golang/core/lib/shared_utils/parsed_git_url_test.go similarity index 84% rename from core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url_test.go rename to api/golang/core/lib/shared_utils/parsed_git_url_test.go index 60ea2b0d25..aa52f9e080 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/parsed_git_url_test.go +++ b/api/golang/core/lib/shared_utils/parsed_git_url_test.go @@ -1,4 +1,4 @@ -package git_package_content_provider +package shared_utils import ( "fmt" @@ -18,7 +18,7 @@ const ( ) func TestParsedGitURL_SimpleParse(t *testing.T) { - parsedURL, err := parseGitURL(githubSampleURL) + parsedURL, err := ParseGitURL(githubSampleURL) require.Nil(t, err) expectedParsedURL := newParsedGitURL( @@ -35,7 +35,7 @@ func TestParsedGitURL_SimpleParse(t *testing.T) { func TestParsedGitURL_FailsOnNonGithubURL(t *testing.T) { nonGithubURL := "kurtosis-git.com/" + testModuleAuthor + "/" + testModuleName + "/" + testFileName - _, err := parseGitURL(nonGithubURL) + _, err := ParseGitURL(nonGithubURL) require.NotNil(t, err) expectedErrorMsg := "We only support modules on Github for now" @@ -46,7 +46,7 @@ func TestParsedGitURL_FailsOnNonGithubURL(t *testing.T) { func TestParsedGitURL_FailsOnNonNonEmptySchema(t *testing.T) { ftpSchema := "ftp" nonGithubURL := ftpSchema + "://github.com/" + testModuleAuthor + "/" + testModuleName + "/" + testFileName - _, err := parseGitURL(nonGithubURL) + _, err := ParseGitURL(nonGithubURL) require.NotNil(t, err) expectedErrorMsg := fmt.Sprintf("Expected schema to be empty got '%v'", ftpSchema) @@ -56,19 +56,19 @@ func TestParsedGitURL_FailsOnNonNonEmptySchema(t *testing.T) { func TestParsedGitURL_IfNoFileThenRelativeFilePathIsEmpty(t *testing.T) { pathWithoutFile := "github.com/" + testModuleAuthor + "/" + testModuleName - parsedURL, err := parseGitURL(pathWithoutFile) + parsedURL, err := ParseGitURL(pathWithoutFile) require.Nil(t, err) require.Equal(t, "", parsedURL.relativeFilePath) } func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) { escapedURLWithoutStartosisFile := "github.com/../etc/passwd" - parsedURL, err := parseGitURL(escapedURLWithoutStartosisFile) + parsedURL, err := ParseGitURL(escapedURLWithoutStartosisFile) require.Nil(t, err) require.Equal(t, "", parsedURL.relativeFilePath) escapedURLWithStartosisFile := "github.com/../../etc/passwd/startosis.star" - parsedURL, err = parseGitURL(escapedURLWithStartosisFile) + parsedURL, err = ParseGitURL(escapedURLWithStartosisFile) require.Nil(t, err) require.Equal(t, parsedURL.moduleAuthor, "etc") require.Equal(t, parsedURL.moduleName, "passwd") @@ -77,7 +77,7 @@ func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) { require.Equal(t, parsedURL.relativeRepoPath, "etc/passwd") escapedURLWithStartosisFile = "github.com/foo/../etc/passwd/startosis.star" - parsedURL, err = parseGitURL(escapedURLWithStartosisFile) + parsedURL, err = ParseGitURL(escapedURLWithStartosisFile) require.Nil(t, err) require.Equal(t, parsedURL.moduleAuthor, "etc") require.Equal(t, parsedURL.moduleName, "passwd") @@ -86,14 +86,14 @@ func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) { require.Equal(t, parsedURL.relativeRepoPath, "etc/passwd") escapedURLWithStartosisFile = "github.com/foo/../etc/../passwd" - _, err = parseGitURL(escapedURLWithStartosisFile) + _, err = ParseGitURL(escapedURLWithStartosisFile) require.NotNil(t, err) expectedErrorMsg := fmt.Sprintf("Error parsing the URL of module: '%s'. The path should contain at least 2 subpaths got '[passwd]'", escapedURLWithStartosisFile) require.Contains(t, err.Error(), expectedErrorMsg) } func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) { - parsedURL, err := parseGitURL(githubSampleUrlWithTag) + parsedURL, err := ParseGitURL(githubSampleUrlWithTag) require.Nil(t, err) expectedParsedURL := newParsedGitURL( @@ -107,7 +107,7 @@ func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) { require.Equal(t, expectedParsedURL, parsedURL) - parsedURL, err = parseGitURL(githubSampleUrlWithBranchContainingVersioningDelimiter) + parsedURL, err = ParseGitURL(githubSampleUrlWithBranchContainingVersioningDelimiter) require.Nil(t, err) expectedParsedURL = newParsedGitURL( @@ -123,46 +123,46 @@ func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) { } func TestParsedGitUrl_ResolvesRelativeUrl(t *testing.T) { - parsedUrl, err := parseGitURL(githubSampleURL) + parsedUrl, err := ParseGitURL(githubSampleURL) require.Nil(t, err) relativeUrl := "./lib.star" - absoluteUrl := parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl) + absoluteUrl := parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl) require.Nil(t, err) expected := "github.com/kurtosis-tech/sample-startosis-load/lib.star" require.Equal(t, expected, absoluteUrl) relativeUrl = "./src/lib.star" - absoluteUrl = parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl) + absoluteUrl = parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl) require.Nil(t, err) expected = "github.com/kurtosis-tech/sample-startosis-load/src/lib.star" require.Equal(t, expected, absoluteUrl) } func TestParsedGitUrl_ResolvesRelativeUrlForUrlWithTag(t *testing.T) { - parsedUrl, err := parseGitURL(githubSampleUrlWithTag) + parsedUrl, err := ParseGitURL(githubSampleUrlWithTag) require.Nil(t, err) relativeUrl := "./lib.star" - absoluteUrl := parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl) + absoluteUrl := parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl) require.Nil(t, err) expected := "github.com/kurtosis-tech/sample-startosis-load/lib.star" require.Equal(t, expected, absoluteUrl) relativeUrl = "./src/lib.star" - absoluteUrl = parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl) + absoluteUrl = parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl) require.Nil(t, err) expected = "github.com/kurtosis-tech/sample-startosis-load/src/lib.star" require.Equal(t, expected, absoluteUrl) } func TestParsedGitUrl_ResolvesWithUrlWithVersionBranchWithSlash(t *testing.T) { - parsedUrl, err := parseGitURL(githubSampleUrlWithVersionWithSlash) + parsedUrl, err := ParseGitURL(githubSampleUrlWithVersionWithSlash) require.Nil(t, err) require.Equal(t, "foo/bar", parsedUrl.tagBranchOrCommit) - parsedUrl, err = parseGitURL(githubSampleUrlWithVersionWithSlashAndFile) + parsedUrl, err = ParseGitURL(githubSampleUrlWithVersionWithSlashAndFile) require.Nil(t, err) require.Equal(t, "foo/bar", parsedUrl.tagBranchOrCommit) require.Equal(t, "kurtosis-tech/sample-startosis-load/main.star", parsedUrl.relativeFilePath) diff --git a/cli/cli/command_str_consts/command_str_consts.go b/cli/cli/command_str_consts/command_str_consts.go index 0041a97edc..9874cb4985 100644 --- a/cli/cli/command_str_consts/command_str_consts.go +++ b/cli/cli/command_str_consts/command_str_consts.go @@ -70,6 +70,8 @@ const ( VersionCmdStr = "version" ImportCmdStr = "import" GatewayCmdStr = "gateway" + PackageCmdStr = "package" + InitCmdStr = "init" PortCmdStr = "port" PortPrintCmdStr = "print" WebCmdStr = "web" diff --git a/cli/cli/commands/package/init_cmd/init_cmd.go b/cli/cli/commands/package/init_cmd/init_cmd.go new file mode 100644 index 0000000000..7adbdd6c49 --- /dev/null +++ b/cli/cli/commands/package/init_cmd/init_cmd.go @@ -0,0 +1,85 @@ +package init_cmd + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" + "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel" + "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args" + "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags" + "github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts" + "github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_package" + "github.com/kurtosis-tech/stacktrace" + "os" +) + +const ( + packageNameArgKey = "package-name" + packageNameArgDefaultValue = "" + packageNameArgIsOptional = false + packageNameArgIsGreedy = false + + executablePackageFlagKey = "main" + executablePackageFlagDefaultValue = "false" +) + +var InitCmd = &lowlevel.LowlevelKurtosisCommand{ + CommandStr: command_str_consts.InitCmdStr, + ShortDescription: "Creates a new Kurtosis package", + LongDescription: "This command initializes the current directory to be a Kurtosis package by creating a `kurtosis.yml` with the given package name.", + Args: []*args.ArgConfig{ + { + Key: packageNameArgKey, + DefaultValue: packageNameArgDefaultValue, + IsOptional: packageNameArgIsOptional, + IsGreedy: packageNameArgIsGreedy, + ValidationFunc: validatePackageNameArg, + }, + }, + Flags: []*flags.FlagConfig{ + { + Key: executablePackageFlagKey, + Usage: "indicates that the created package is an executable package, and generates a 'main.star' if one does not already exist.", + Type: flags.FlagType_Bool, + Default: executablePackageFlagDefaultValue, + }, + }, + PreValidationAndRunFunc: nil, + RunFunc: run, + PostValidationAndRunFunc: nil, +} + +func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) error { + packageNameArg, err := args.GetNonGreedyArg(packageNameArgKey) + if err != nil { + return stacktrace.Propagate(err, "an error occurred getting the value of argument with key '%v'", packageNameArgKey) + } + + executablePackageFlag, err := flags.GetBool(executablePackageFlagKey) + if err != nil { + return stacktrace.Propagate(err, "an error occurred getting the value of flag '%v'", executablePackageFlagKey) + } + + packageDestinationDirpath, err := os.Getwd() + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting the current working directory for creating the Kurtosis package") + } + + if err := kurtosis_package.InitializeKurtosisPackage(packageDestinationDirpath, packageNameArg, executablePackageFlag); err != nil { + return stacktrace.Propagate(err, "An error occurred initializing the Kurtosis package '%s' in '%s'", packageNameArg, packageDestinationDirpath) + } + + return nil +} + +func validatePackageNameArg(_ context.Context, _ *flags.ParsedFlags, args *args.ParsedArgs) error { + packageNameArg, err := args.GetNonGreedyArg(packageNameArgKey) + if err != nil { + return stacktrace.Propagate(err, "an error occurred getting the value of argument with key '%v'", packageNameArgKey) + } + + if _, err := shared_utils.ParseGitURL(packageNameArg); err != nil { + return stacktrace.Propagate(err, "An erro occurred validating package name '%v', invalid GitHub URL", packageNameArg) + } + + return nil +} diff --git a/cli/cli/commands/package/package.go b/cli/cli/commands/package/package.go new file mode 100644 index 0000000000..36b6036803 --- /dev/null +++ b/cli/cli/commands/package/package.go @@ -0,0 +1,19 @@ +package _package + +import ( + "github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts" + "github.com/kurtosis-tech/kurtosis/cli/cli/commands/package/init_cmd" + "github.com/spf13/cobra" +) + +// PackageCmd Suppressing exhaustruct requirement because this struct has ~40 properties +// nolint: exhaustruct +var PackageCmd = &cobra.Command{ + Use: command_str_consts.PackageCmdStr, + Short: "Manage packages", + RunE: nil, +} + +func init() { + PackageCmd.AddCommand(init_cmd.InitCmd.MustGetCobraCommand()) +} diff --git a/cli/cli/commands/root.go b/cli/cli/commands/root.go index 820e0423d4..d27ed2a8d0 100644 --- a/cli/cli/commands/root.go +++ b/cli/cli/commands/root.go @@ -27,6 +27,7 @@ import ( "github.com/kurtosis-tech/kurtosis/cli/cli/commands/kurtosis_context" "github.com/kurtosis-tech/kurtosis/cli/cli/commands/lint" "github.com/kurtosis-tech/kurtosis/cli/cli/commands/lsp" + _package "github.com/kurtosis-tech/kurtosis/cli/cli/commands/package" "github.com/kurtosis-tech/kurtosis/cli/cli/commands/port" "github.com/kurtosis-tech/kurtosis/cli/cli/commands/portal" "github.com/kurtosis-tech/kurtosis/cli/cli/commands/run" @@ -130,6 +131,7 @@ func init() { RootCmd.AddCommand(twitter.TwitterCmd.MustGetCobraCommand()) RootCmd.AddCommand(version.VersionCmd) RootCmd.AddCommand(web.WebCmd.MustGetCobraCommand()) + RootCmd.AddCommand(_package.PackageCmd) } // ==================================================================================================== diff --git a/cli/cli/kurtosis_config/kurtosis_config_store.go b/cli/cli/kurtosis_config/kurtosis_config_store.go index c4eb34687e..e662f7e8f3 100644 --- a/cli/cli/kurtosis_config/kurtosis_config_store.go +++ b/cli/cli/kurtosis_config/kurtosis_config_store.go @@ -1,7 +1,6 @@ package kurtosis_config import ( - "github.com/go-yaml/yaml" "github.com/kurtosis-tech/kurtosis/cli/cli/helpers/host_machine_directories" "github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/config_version" "github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/overrides_deserializers" @@ -9,6 +8,7 @@ import ( "github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/resolved_config" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" "io" "os" "sync" diff --git a/cli/cli/kurtosis_package/kurtosis_package.go b/cli/cli/kurtosis_package/kurtosis_package.go new file mode 100644 index 0000000000..aad6888721 --- /dev/null +++ b/cli/cli/kurtosis_package/kurtosis_package.go @@ -0,0 +1,104 @@ +package kurtosis_package + +import ( + "fmt" + "github.com/go-yaml/yaml" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "os" + "path" +) + +const ( + kurtosisPackageFilePermissions os.FileMode = 0644 + kurtosisYmlFilename = "kurtosis.yml" + mainStarFilename = "main.star" + mainStarFileContentStr = `def run(plan): + # TODO + plan.print("hello world!")` + kurtosisYmlDescriptionFormat = "# %s\nEnter description Markdown here." +) + +func InitializeKurtosisPackage(packageDirpath string, packageName string, isExecutablePackage bool) error { + + // validate package name + _, err := shared_utils.ParseGitURL(packageName) + if err != nil { + return stacktrace.Propagate(err, "An erro occurred validating package name '%v', invalid GitHub URL", packageName) + } + + logrus.Debugf("Initializaing the '%s' Kurtosis package...", packageName) + if err := createKurtosisYamlFile(packageDirpath, packageName); err != nil { + return stacktrace.Propagate(err, "An error occurred creating the '%s' on '%s'", kurtosisYmlFilename, packageDirpath) + } + + if isExecutablePackage { + if err := createMainStarFile(packageDirpath); err != nil { + return stacktrace.Propagate(err, "An error occurred creating the '%s' on '%s'", mainStarFilename, packageDirpath) + } + } + + logrus.Debugf("...'%s' Kurtosis package successfully initialized", packageName) + + return nil +} + +func createKurtosisYamlFile(packageDirpath string, packageName string) error { + + defaultPackageDescriptionForInitPackage := fmt.Sprintf(kurtosisYmlDescriptionFormat, packageName) + defaultPackageReplaceOptionsForInitPackage := map[string]string{} + + kurtosisYaml := enclaves.NewKurtosisYaml(packageName, defaultPackageDescriptionForInitPackage, defaultPackageReplaceOptionsForInitPackage) + + kurtosisYAMLContent, err := yaml.Marshal(kurtosisYaml) + if err != nil { + return stacktrace.Propagate(err, "An error occurred marshalling Kurtosis yaml file '%+v'", kurtosisYaml) + } + + kurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + + fileInfo, err := os.Stat(kurtosisYamlFilepath) + if fileInfo != nil { + return stacktrace.NewError("Imposible to create a new Kurtosis package inside '%s' because a file with name '%s' already exist on this path", packageDirpath, kurtosisYmlFilename) + } + if os.IsNotExist(err) { + logrus.Debugf("Creating the '%s' file...", kurtosisYmlFilename) + err = os.WriteFile(kurtosisYamlFilepath, kurtosisYAMLContent, kurtosisPackageFilePermissions) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing the '%s' file", kurtosisYamlFilepath) + } + logrus.Debugf("...'%s' file created", kurtosisYmlFilename) + } + if err != nil { + return stacktrace.Propagate(err, "An error occurred creating the '%s' file on '%s'", kurtosisYmlFilename, packageDirpath) + } + + return nil +} + +func createMainStarFile(packageDirpath string) error { + + mainStarFilepath := path.Join(packageDirpath, mainStarFilename) + + fileInfo, err := os.Stat(mainStarFilepath) + if fileInfo != nil { + return stacktrace.NewError("Imposible to create a new Kurtosis package inside '%s' because a file with name '%s' already exist on this path", packageDirpath, mainStarFilename) + } + + mainStarFileContent := []byte(mainStarFileContentStr) + + if os.IsNotExist(err) { + logrus.Debugf("Creating the '%s' file...", mainStarFilename) + err = os.WriteFile(mainStarFilepath, mainStarFileContent, kurtosisPackageFilePermissions) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing the '%s' file", mainStarFilepath) + } + logrus.Debugf("...'%s' file created", mainStarFilename) + } + if err != nil { + return stacktrace.Propagate(err, "An error occurred creating the '%s' file on '%s'", mainStarFilename, packageDirpath) + } + return nil +} diff --git a/cli/cli/kurtosis_package/kurtosis_package_test.go b/cli/cli/kurtosis_package/kurtosis_package_test.go new file mode 100644 index 0000000000..165d7624cd --- /dev/null +++ b/cli/cli/kurtosis_package/kurtosis_package_test.go @@ -0,0 +1,184 @@ +package kurtosis_package + +import ( + "github.com/go-yaml/yaml" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" + "github.com/kurtosis-tech/stacktrace" + "github.com/stretchr/testify/require" + "os" + "path" + "testing" +) + +const ( + isExecutablePackage = true + isNotExecutablePackage = false + myTestPackage = "github.com/org/my-package" + mySecondTestPackage = "github.com/org/second-package" + packageInitializeTestDirPattern = "package-initialize-test-dir-*" +) + +func TestInitializeKurtosisPackage_Success(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + packageName := myTestPackage + err = InitializeKurtosisPackage(packageDirpath, packageName, isExecutablePackage) + require.NoError(t, err) + + // check kurtosis.yml file creation + expectedKurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + fileBytes, err := os.ReadFile(expectedKurtosisYamlFilepath) + require.NoError(t, err) + + kurtosisYamlObj := &enclaves.KurtosisYaml{ + PackageName: "", + PackageDescription: "", + PackageReplaceOptions: map[string]string{}, + } + err = yaml.UnmarshalStrict(fileBytes, kurtosisYamlObj) + require.NoError(t, err) + require.Equal(t, packageName, kurtosisYamlObj.PackageName) + + // check main.star file creation + expectedMainStarFilepath := path.Join(packageDirpath, mainStarFilename) + fileBytes, err = os.ReadFile(expectedMainStarFilepath) + require.NoError(t, err) + require.Equal(t, mainStarFileContentStr, string(fileBytes)) +} + +func TestInitializeKurtosisPackage_IsNotExecutablePackageSuccess(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + packageName := myTestPackage + err = InitializeKurtosisPackage(packageDirpath, packageName, isNotExecutablePackage) + require.NoError(t, err) + + // check kurtosis.yml file creation + expectedKurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + fileBytes, err := os.ReadFile(expectedKurtosisYamlFilepath) + require.NoError(t, err) + + kurtosisYamlObj := &enclaves.KurtosisYaml{ + PackageName: "", + PackageDescription: "", + PackageReplaceOptions: map[string]string{}, + } + err = yaml.UnmarshalStrict(fileBytes, kurtosisYamlObj) + require.NoError(t, err) + require.Equal(t, packageName, kurtosisYamlObj.PackageName) + + // check main.star file was not created + expectedMainStarFilepath := path.Join(packageDirpath, mainStarFilename) + fileBytes, err = os.ReadFile(expectedMainStarFilepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + require.Nil(t, fileBytes) +} + +func TestInitializeKurtosisPackage_FailsIfKurtosisYmlAlreadyExist(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + packageName := myTestPackage + err = InitializeKurtosisPackage(packageDirpath, packageName, isExecutablePackage) + require.NoError(t, err) + + expectedKurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + fileBytes, err := os.ReadFile(expectedKurtosisYamlFilepath) + require.NoError(t, err) + + secondPackageName := mySecondTestPackage + err = InitializeKurtosisPackage(packageDirpath, secondPackageName, isExecutablePackage) + require.Error(t, err) + expectedErrorMsgPortion := "'kurtosis.yml' already exist on this path" + require.Contains(t, stacktrace.RootCause(err).Error(), expectedErrorMsgPortion) + + kurtosisYamlObj := &enclaves.KurtosisYaml{ + PackageName: "", + PackageDescription: "", + PackageReplaceOptions: map[string]string{}, + } + + err = yaml.UnmarshalStrict(fileBytes, kurtosisYamlObj) + require.NoError(t, err) + + require.Equal(t, packageName, kurtosisYamlObj.PackageName) +} + +func TestInitializeKurtosisPackage_FailsIfMainStarFileAlreadyExist(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + // there is a main.star file inside the folder before initializing the package + mainStarFilepath := path.Join(packageDirpath, mainStarFilename) + + _, err = os.Create(mainStarFilepath) + require.NoError(t, err) + + packageName := myTestPackage + err = InitializeKurtosisPackage(packageDirpath, packageName, isNotExecutablePackage) + require.NoError(t, err) + + // check kurtosis.yml file creation + expectedKurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + fileBytes, err := os.ReadFile(expectedKurtosisYamlFilepath) + require.NoError(t, err) + + kurtosisYamlObj := &enclaves.KurtosisYaml{ + PackageName: "", + PackageDescription: "", + PackageReplaceOptions: map[string]string{}, + } + err = yaml.UnmarshalStrict(fileBytes, kurtosisYamlObj) + require.NoError(t, err) + require.Equal(t, packageName, kurtosisYamlObj.PackageName) + +} + +func TestInitializeKurtosisPackage_InvalidPackageNameError(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + packageName := "my-url/org/my-package" + err = InitializeKurtosisPackage(packageDirpath, packageName, isExecutablePackage) + require.Error(t, err) + require.ErrorContains(t, err, "invalid GitHub URL") +} + +func TestInitializeKurtosisPackage_MakeScriptPackageSuccess(t *testing.T) { + packageDirpath, err := os.MkdirTemp("", packageInitializeTestDirPattern) + require.NoError(t, err) + defer os.RemoveAll(packageDirpath) + + packageName := myTestPackage + err = InitializeKurtosisPackage(packageDirpath, packageName, isExecutablePackage) + require.NoError(t, err) + + expectedKurtosisYamlFilepath := path.Join(packageDirpath, kurtosisYmlFilename) + fileBytes, err := os.ReadFile(expectedKurtosisYamlFilepath) + require.NoError(t, err) + + secondPackageName := mySecondTestPackage + err = InitializeKurtosisPackage(packageDirpath, secondPackageName, isExecutablePackage) + require.Error(t, err) + expectedErrorMsgPortion := "'kurtosis.yml' already exist on this path" + require.Contains(t, stacktrace.RootCause(err).Error(), expectedErrorMsgPortion) + + kurtosisYamlObj := &enclaves.KurtosisYaml{ + PackageName: "", + PackageDescription: "", + PackageReplaceOptions: map[string]string{}, + } + + err = yaml.UnmarshalStrict(fileBytes, kurtosisYamlObj) + require.NoError(t, err) + + require.Equal(t, packageName, kurtosisYamlObj.PackageName) +} diff --git a/core/server/api_container/server/startosis_engine/startosis_constants/startosis_constants.go b/core/server/api_container/server/startosis_engine/startosis_constants/startosis_constants.go index 90c30a7b86..5262687ccd 100644 --- a/core/server/api_container/server/startosis_engine/startosis_constants/startosis_constants.go +++ b/core/server/api_container/server/startosis_engine/startosis_constants/startosis_constants.go @@ -3,10 +3,9 @@ package startosis_constants type StarlarkContextParam string const ( - MainFileName = "main.star" - KurtosisYamlName = "kurtosis.yml" - GithubDomainPrefix = "github.com" - EmptyInputArgs = "{}" // empty JSON + MainFileName = "main.star" + KurtosisYamlName = "kurtosis.yml" + EmptyInputArgs = "{}" // empty JSON NoOutputObject = "" diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index 337ae2910b..5a0849661b 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" @@ -21,6 +22,7 @@ const ( temporaryRepoDirPattern = "tmp-repo-dir-*" temporaryArchiveFilePattern = "temp-module-archive-*.tgz" defaultTmpDir = "" + emptyTagBranchOrCommit = "" onlyOneReplacement = 1 replacedWithEmptyString = "" @@ -56,12 +58,12 @@ func NewGitPackageContentProvider(moduleDir string, tmpDir string, enclaveDb *en } func (provider *GitPackageContentProvider) ClonePackage(packageId string) (string, *startosis_errors.InterpretationError) { - parsedURL, interpretationError := parseGitURL(packageId) - if interpretationError != nil { - return "", interpretationError + parsedURL, err := shared_utils.ParseGitURL(packageId) + if err != nil { + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for package ID '%s'", packageId) } - interpretationError = provider.atomicClone(parsedURL) + interpretationError := provider.atomicClone(parsedURL) if interpretationError != nil { return "", interpretationError } @@ -88,15 +90,15 @@ func (provider *GitPackageContentProvider) GetKurtosisYaml(packageAbsolutePathOn } func (provider *GitPackageContentProvider) GetOnDiskAbsoluteFilePath(absoluteFileLocator string) (string, *startosis_errors.InterpretationError) { - parsedURL, interpretationError := parseGitURL(absoluteFileLocator) - if interpretationError != nil { - return "", interpretationError + parsedURL, err := shared_utils.ParseGitURL(absoluteFileLocator) + if err != nil { + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for absolute file locator '%s'", absoluteFileLocator) } - if parsedURL.relativeFilePath == "" { + if parsedURL.GetRelativeFilePath() == "" { return "", startosis_errors.NewInterpretationError("The path '%v' needs to point to a specific file but it didn't. Users can only read or import specific files and not entire packages.", absoluteFileLocator) } - pathToFileOnDisk := path.Join(provider.packagesDir, parsedURL.relativeFilePath) - packagePath := path.Join(provider.packagesDir, parsedURL.relativeRepoPath) + pathToFileOnDisk := path.Join(provider.packagesDir, parsedURL.GetRelativeFilePath()) + packagePath := path.Join(provider.packagesDir, parsedURL.GetRelativeRepoPath()) // Return the file path straight if it exists if _, err := os.Stat(pathToFileOnDisk); err == nil { @@ -106,19 +108,19 @@ func (provider *GitPackageContentProvider) GetOnDiskAbsoluteFilePath(absoluteFil // Check if the repo exists // If the repo exists but the `pathToFileOnDisk` doesn't that means there's a mistake in the locator if _, err := os.Stat(packagePath); err == nil { - relativeFilePathWithoutPackageName := strings.Replace(parsedURL.relativeFilePath, parsedURL.relativeRepoPath, replacedWithEmptyString, onlyOneReplacement) - return "", startosis_errors.NewInterpretationError("'%v' doesn't exist in the package '%v'", relativeFilePathWithoutPackageName, parsedURL.relativeRepoPath) + relativeFilePathWithoutPackageName := strings.Replace(parsedURL.GetRelativeFilePath(), parsedURL.GetRelativeRepoPath(), replacedWithEmptyString, onlyOneReplacement) + return "", startosis_errors.NewInterpretationError("'%v' doesn't exist in the package '%v'", relativeFilePathWithoutPackageName, parsedURL.GetRelativeRepoPath()) } // Otherwise clone the repo and return the absolute path of the requested file - interpretationError = provider.atomicClone(parsedURL) + interpretationError := provider.atomicClone(parsedURL) if interpretationError != nil { return "", interpretationError } // check whether kurtosis yaml exists in the path - maybeKurtosisYamlPath, err := getKurtosisYamlPathForFileUrl(pathToFileOnDisk, provider.packagesDir) - if err != nil { + maybeKurtosisYamlPath, interpretationError := getKurtosisYamlPathForFileUrl(pathToFileOnDisk, provider.packagesDir) + if interpretationError != nil { return "", startosis_errors.WrapWithInterpretationError(err, "Error occurred while verifying whether '%v' belongs to a Kurtosis package.", absoluteFileLocator) } @@ -149,15 +151,15 @@ func (provider *GitPackageContentProvider) GetModuleContents(fileInsideModuleUrl } func (provider *GitPackageContentProvider) GetOnDiskAbsolutePackagePath(packageId string) (string, *startosis_errors.InterpretationError) { - parsedPackageId, interpretationError := parseGitURL(packageId) - if interpretationError != nil { - return "", interpretationError + parsedPackageId, err := shared_utils.ParseGitURL(packageId) + if err != nil { + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for package ID '%s'", packageId) } relPackagePathToPackagesDir := getPathToPackageRoot(parsedPackageId) packageAbsolutePathOnDisk := path.Join(provider.packagesDir, relPackagePathToPackagesDir) - _, err := os.Stat(packageAbsolutePathOnDisk) + _, err = os.Stat(packageAbsolutePathOnDisk) if err != nil { return "", startosis_errors.NewInterpretationError("Package '%v' does not exist on disk at '%s'", packageId, packageAbsolutePathOnDisk) } @@ -165,9 +167,9 @@ func (provider *GitPackageContentProvider) GetOnDiskAbsolutePackagePath(packageI } func (provider *GitPackageContentProvider) StorePackageContents(packageId string, moduleTar io.Reader, overwriteExisting bool) (string, *startosis_errors.InterpretationError) { - parsedPackageId, interpretationError := parseGitURL(packageId) - if interpretationError != nil { - return "", interpretationError + parsedPackageId, err := shared_utils.ParseGitURL(packageId) + if err != nil { + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred parsing Git URL for package ID '%s'", packageId) } relPackagePathToPackagesDir := getPathToPackageRoot(parsedPackageId) @@ -180,7 +182,7 @@ func (provider *GitPackageContentProvider) StorePackageContents(packageId string } } - _, err := os.Stat(packageAbsolutePathOnDisk) + _, err = os.Stat(packageAbsolutePathOnDisk) if err == nil { return "", startosis_errors.NewInterpretationError("Package '%v' already exists on disk, not overwriting", packageAbsolutePathOnDisk) } @@ -215,16 +217,16 @@ func (provider *GitPackageContentProvider) GetAbsoluteLocatorForRelativeLocator( } // maybe it's not a relative url in which case we return the url - _, errorParsingUrl := parseGitURL(maybeRelativeLocator) + _, errorParsingUrl := shared_utils.ParseGitURL(maybeRelativeLocator) if errorParsingUrl == nil { absoluteLocator = maybeRelativeLocator } else { - parsedParentModuleId, errorParsingPackageId := parseGitURL(parentModuleId) + parsedParentModuleId, errorParsingPackageId := shared_utils.ParseGitURL(parentModuleId) if errorParsingPackageId != nil { return "", startosis_errors.NewInterpretationError("Parent package id '%v' isn't a valid locator; relative URLs don't work with standalone scripts", parentModuleId) } - absoluteLocator = parsedParentModuleId.getAbsoluteLocatorRelativeToThisURL(maybeRelativeLocator) + absoluteLocator = parsedParentModuleId.GetAbsoluteLocatorRelativeToThisURL(maybeRelativeLocator) } replacedAbsoluteLocator := replaceAbsoluteLocator(absoluteLocator, packageReplaceOptions) @@ -243,7 +245,7 @@ func (provider *GitPackageContentProvider) CloneReplacedPackagesIfNeeded(current shouldClonePackage := false - isExistingLocalReplace := isLocalLocator(existingReplace) + isExistingLocalReplace := isLocalDependencyReplace(existingReplace) logrus.Debugf("existingReplace '%v' isExistingLocalReplace? '%v', ", existingReplace, isExistingLocalReplace) currentReplace, isCurrentReplace := currentPackageReplaceOptions[packageId] @@ -281,22 +283,22 @@ func (provider *GitPackageContentProvider) CloneReplacedPackagesIfNeeded(current // atomicClone This first clones to a temporary directory and then moves it // TODO make this support versioning via tags, commit hashes or branches -func (provider *GitPackageContentProvider) atomicClone(parsedURL *ParsedGitURL) *startosis_errors.InterpretationError { +func (provider *GitPackageContentProvider) atomicClone(parsedURL *shared_utils.ParsedGitURL) *startosis_errors.InterpretationError { // First we clone into a temporary directory tempRepoDirPath, err := os.MkdirTemp(provider.packagesTmpDir, temporaryRepoDirPattern) if err != nil { - return startosis_errors.WrapWithInterpretationError(err, "Cloning the module '%s' failed. Error creating temporary directory for the repository to be cloned into", parsedURL.gitURL) + return startosis_errors.WrapWithInterpretationError(err, "Cloning the module '%s' failed. Error creating temporary directory for the repository to be cloned into", parsedURL.GetGitURL()) } defer os.RemoveAll(tempRepoDirPath) - gitClonePath := path.Join(tempRepoDirPath, parsedURL.relativeRepoPath) + gitClonePath := path.Join(tempRepoDirPath, parsedURL.GetRelativeRepoPath()) depth := defaultDepth - if parsedURL.tagBranchOrCommit != emptyTagBranchOrCommit { + if parsedURL.GetTagBranchOrCommit() != emptyTagBranchOrCommit { depth = depthAssumingBranchTagsCommitsAreSpecified } repo, err := git.PlainClone(gitClonePath, isNotBareClone, &git.CloneOptions{ - URL: parsedURL.gitURL, + URL: parsedURL.GetGitURL(), Auth: nil, RemoteName: "", ReferenceName: "", @@ -313,11 +315,11 @@ func (provider *GitPackageContentProvider) atomicClone(parsedURL *ParsedGitURL) // TODO remove public repository from error after we support private repositories // We silent the underlying error here as it can be confusing to the user. For example, when there's a typo in // the repo name, pointing to a non existing repo, the underlying error is: "authentication required" - logrus.Errorf("Error cloning git repository: '%s' to '%s'. Error was: \n%s", parsedURL.gitURL, gitClonePath, err.Error()) - return startosis_errors.NewInterpretationError("Error in cloning git repository '%s' to '%s'. Make sure that '%v' exists and is a public repository.", parsedURL.gitURL, gitClonePath, parsedURL.gitURL) + logrus.Errorf("Error cloning git repository: '%s' to '%s'. Error was: \n%s", parsedURL.GetGitURL(), gitClonePath, err.Error()) + return startosis_errors.NewInterpretationError("Error in cloning git repository '%s' to '%s'. Make sure that '%v' exists and is a public repository.", parsedURL.GetGitURL(), gitClonePath, parsedURL.GetGitURL()) } - if parsedURL.tagBranchOrCommit != emptyTagBranchOrCommit { + if parsedURL.GetTagBranchOrCommit() != emptyTagBranchOrCommit { referenceTagOrBranch, found, referenceErr := getReferenceName(repo, parsedURL) if err != nil { return referenceErr @@ -334,77 +336,77 @@ func (provider *GitPackageContentProvider) atomicClone(parsedURL *ParsedGitURL) // if we have a tag or branch we set it checkoutOptions.Branch = referenceTagOrBranch } else { - maybeHash := plumbing.NewHash(parsedURL.tagBranchOrCommit) + maybeHash := plumbing.NewHash(parsedURL.GetTagBranchOrCommit()) // check whether the hash is a valid hash _, err = repo.CommitObject(maybeHash) if err != nil { - return startosis_errors.NewInterpretationError("Tried using the passed version '%v' as commit as we couldn't find a tag/branch for it in the repo '%v' but failed", parsedURL.tagBranchOrCommit, parsedURL.gitURL) + return startosis_errors.NewInterpretationError("Tried using the passed version '%v' as commit as we couldn't find a tag/branch for it in the repo '%v' but failed", parsedURL.GetTagBranchOrCommit(), parsedURL.GetGitURL()) } checkoutOptions.Hash = maybeHash } workTree, err := repo.Worktree() if err != nil { - return startosis_errors.NewInterpretationError("Tried getting worktree for cloned repo '%v' but failed", parsedURL.gitURL) + return startosis_errors.NewInterpretationError("Tried getting worktree for cloned repo '%v' but failed", parsedURL.GetGitURL()) } if err := workTree.Checkout(checkoutOptions); err != nil { - return startosis_errors.NewInterpretationError("Tried checking out '%v' on repository '%v' but failed", parsedURL.tagBranchOrCommit, parsedURL.gitURL) + return startosis_errors.NewInterpretationError("Tried checking out '%v' on repository '%v' but failed", parsedURL.GetTagBranchOrCommit(), parsedURL.GetGitURL()) } } // Then we move it into the target directory - packageAuthorPath := path.Join(provider.packagesDir, parsedURL.moduleAuthor) - packagePath := path.Join(provider.packagesDir, parsedURL.relativeRepoPath) + packageAuthorPath := path.Join(provider.packagesDir, parsedURL.GetModuleAuthor()) + packagePath := path.Join(provider.packagesDir, parsedURL.GetRelativeRepoPath()) fileMode, err := os.Stat(packageAuthorPath) if err == nil && !fileMode.IsDir() { return startosis_errors.WrapWithInterpretationError(err, "Expected '%s' to be a directory but it is something else", packageAuthorPath) } if err != nil { if err = os.Mkdir(packageAuthorPath, moduleDirPermission); err != nil { - return startosis_errors.WrapWithInterpretationError(err, "Cloning the package '%s' failed. An error occurred while creating the directory '%s'.", parsedURL.gitURL, packageAuthorPath) + return startosis_errors.WrapWithInterpretationError(err, "Cloning the package '%s' failed. An error occurred while creating the directory '%s'.", parsedURL.GetGitURL(), packageAuthorPath) } } if _, err = os.Stat(packagePath); !os.IsNotExist(err) { - logrus.Debugf("Package '%s' already exists in enclave at path '%s'. Going to remove previous version.", parsedURL.gitURL, packagePath) + logrus.Debugf("Package '%s' already exists in enclave at path '%s'. Going to remove previous version.", parsedURL.GetGitURL(), packagePath) if err = os.RemoveAll(packagePath); err != nil { - return startosis_errors.NewInterpretationError("Unable to remove a previous version of package '%s' existing inside Kurtosis enclave at '%s'.", parsedURL.gitURL, packagePath) + return startosis_errors.NewInterpretationError("Unable to remove a previous version of package '%s' existing inside Kurtosis enclave at '%s'.", parsedURL.GetGitURL(), packagePath) } } if err = os.Rename(gitClonePath, packagePath); err != nil { - return startosis_errors.NewInterpretationError("Cloning the package '%s' failed. An error occurred while moving package at temporary destination '%s' to final destination '%s'", parsedURL.gitURL, gitClonePath, packagePath) + return startosis_errors.NewInterpretationError("Cloning the package '%s' failed. An error occurred while moving package at temporary destination '%s' to final destination '%s'", parsedURL.GetGitURL(), gitClonePath, packagePath) } return nil } // methods checks whether the root of the package is same as repository root // or it is a sub-folder under it -func getPathToPackageRoot(parsedPackagePath *ParsedGitURL) string { - packagePathOnDisk := parsedPackagePath.relativeRepoPath - if parsedPackagePath.relativeFilePath != relativeFilePathNotFound { - packagePathOnDisk = parsedPackagePath.relativeFilePath +func getPathToPackageRoot(parsedPackagePath *shared_utils.ParsedGitURL) string { + packagePathOnDisk := parsedPackagePath.GetRelativeRepoPath() + if parsedPackagePath.GetRelativeFilePath() != relativeFilePathNotFound { + packagePathOnDisk = parsedPackagePath.GetRelativeFilePath() } return packagePathOnDisk } -func getReferenceName(repo *git.Repository, parsedURL *ParsedGitURL) (plumbing.ReferenceName, bool, *startosis_errors.InterpretationError) { - tag, err := repo.Tag(parsedURL.tagBranchOrCommit) +func getReferenceName(repo *git.Repository, parsedURL *shared_utils.ParsedGitURL) (plumbing.ReferenceName, bool, *startosis_errors.InterpretationError) { + tag, err := repo.Tag(parsedURL.GetTagBranchOrCommit()) if err == nil { return tag.Name(), true, nil } if err != nil && err != git.ErrTagNotFound { - return "", false, startosis_errors.NewInterpretationError("An error occurred while checking whether '%v' is a tag and exists in '%v'", parsedURL.tagBranchOrCommit, parsedURL.gitURL) + return "", false, startosis_errors.NewInterpretationError("An error occurred while checking whether '%v' is a tag and exists in '%v'", parsedURL.GetTagBranchOrCommit(), parsedURL.GetGitURL()) } // for branches there is repo.Branch() but that doesn't work with remote branches refs, err := repo.References() if err != nil { - return "", false, startosis_errors.NewInterpretationError("An error occurred while fetching references for repository '%v'", parsedURL.gitURL) + return "", false, startosis_errors.NewInterpretationError("An error occurred while fetching references for repository '%v'", parsedURL.GetGitURL()) } var branchReference *plumbing.Reference err = refs.ForEach(func(r *plumbing.Reference) error { - if r.Name() == plumbing.NewRemoteReferenceName(git.DefaultRemoteName, parsedURL.tagBranchOrCommit) { + if r.Name() == plumbing.NewRemoteReferenceName(git.DefaultRemoteName, parsedURL.GetTagBranchOrCommit()) { branchReference = r return nil } @@ -413,7 +415,7 @@ func getReferenceName(repo *git.Repository, parsedURL *ParsedGitURL) (plumbing.R // checking the error even though the above can't ever error if err != nil { - return "", false, startosis_errors.NewInterpretationError("An error occurred while iterating through references in repository '%v'; This is a bug in Kurtosis", parsedURL.gitURL) + return "", false, startosis_errors.NewInterpretationError("An error occurred while iterating through references in repository '%v'; This is a bug in Kurtosis", parsedURL.GetGitURL()) } if branchReference != nil { @@ -441,15 +443,15 @@ func validateAndGetKurtosisYaml(absPathToKurtosisYmlInThePackage string, package // this method validates whether the package name found in kurtosis yml is same as the location where kurtosis.yml is found func validatePackageNameMatchesKurtosisYamlLocation(kurtosisYaml *yaml_parser.KurtosisYaml, absPathToKurtosisYmlInThePackage string, packageDir string) *startosis_errors.InterpretationError { // get package name from absolute path to package - packageNameFromAbsPackagePath := strings.Replace(absPathToKurtosisYmlInThePackage, packageDir, startosis_constants.GithubDomainPrefix, replaceCountPackageDirWithGithubConstant) + packageNameFromAbsPackagePath := strings.Replace(absPathToKurtosisYmlInThePackage, packageDir, shared_utils.GithubDomainPrefix, replaceCountPackageDirWithGithubConstant) packageName := kurtosisYaml.GetPackageName() if strings.HasSuffix(packageName, osPathSeparatorString) { return startosis_errors.NewInterpretationError("Kurtosis package name cannot have trailing %q; package name: %v and kurtosis.yml is found at: %v", osPathSeparatorString, packageName, packageNameFromAbsPackagePath) } - // re-using parseGitURL with packageName found from kurtosis.yml as it already does some validations - _, err := parseGitURL(packageName) + // re-using ParseGitURL with packageName found from kurtosis.yml as it already does some validations + _, err := shared_utils.ParseGitURL(packageName) if err != nil { return startosis_errors.WrapWithInterpretationError(err, "Error occurred while validating package name: %v which is found in kurtosis.yml at: '%v'", kurtosisYaml.GetPackageName(), packageNameFromAbsPackagePath) } @@ -506,3 +508,10 @@ func getKurtosisYamlPathForFileUrlInternal(absPathToFile string, packagesDir str } return filePathToKurtosisYamlNotFound, nil } + +func isLocalDependencyReplace(replace string) bool { + if strings.HasPrefix(replace, osPathSeparatorString) || strings.HasPrefix(replace, dotRelativePathIndicatorString) { + return true + } + return false +} diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go index 964464c5c8..d605565c88 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider_test.go @@ -2,6 +2,7 @@ package git_package_content_provider import ( "fmt" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" @@ -315,13 +316,13 @@ func TestGetAbsoluteLocatorForRelativeModuleLocator_LocalPackagehReplaceSucceeds func Test_getPathToPackageRoot(t *testing.T) { githubUrlWithKurtosisPackageInSubfolder := "github.com/sample/sample-package/folder/subpackage" - parsedGitUrl, err := parseGitURL(githubUrlWithKurtosisPackageInSubfolder) + parsedGitUrl, err := shared_utils.ParseGitURL(githubUrlWithKurtosisPackageInSubfolder) require.Nil(t, err, "Unexpected error occurred while parsing git url") actual := getPathToPackageRoot(parsedGitUrl) require.Equal(t, "sample/sample-package/folder/subpackage", actual) githubUrlWithRootKurtosisPackage := "github.com/sample/sample-package" - parsedGitUrl, err = parseGitURL(githubUrlWithRootKurtosisPackage) + parsedGitUrl, err = shared_utils.ParseGitURL(githubUrlWithRootKurtosisPackage) require.Nil(t, err, "Unexpected error occurred while parsing git url") actual = getPathToPackageRoot(parsedGitUrl) require.Equal(t, "sample/sample-package", actual) diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go index 5429424e01..491104d947 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go @@ -1,6 +1,7 @@ package git_package_content_provider import ( + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" "github.com/sirupsen/logrus" "strings" ) @@ -49,13 +50,13 @@ func findPackageReplace(absoluteLocator string, packageReplaceOptions map[string pathToAnalyze := absoluteLocator for { - numberSlashes := strings.Count(pathToAnalyze, urlPathSeparator) + numberSlashes := strings.Count(pathToAnalyze, shared_utils.UrlPathSeparator) // check for the minimal path e.g.: github.com/org/package - if numberSlashes < minimumSubPathsForValidGitURL { + if numberSlashes < shared_utils.MinimumSubPathsForValidGitURL { break } - lastIndex := strings.LastIndex(pathToAnalyze, urlPathSeparator) + lastIndex := strings.LastIndex(pathToAnalyze, shared_utils.UrlPathSeparator) packageToBeReplaced := pathToAnalyze[:lastIndex] replaceWithPackage, ok := packageReplaceOptions[packageToBeReplaced] diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go index dce7a7072e..65cbb3f723 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider/mock_package_content_provider.go @@ -1,7 +1,7 @@ package mock_package_content_provider import ( - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" "github.com/kurtosis-tech/kurtosis/core/server/commons/yaml_parser" "github.com/kurtosis-tech/stacktrace" @@ -81,7 +81,7 @@ func (provider *MockPackageContentProvider) GetAbsoluteLocatorForRelativeLocator return "", startosis_errors.NewInterpretationError("Cannot use local absolute locators") } - if strings.HasPrefix(relativeOrAbsoluteModulePath, startosis_constants.GithubDomainPrefix) { + if strings.HasPrefix(relativeOrAbsoluteModulePath, shared_utils.GithubDomainPrefix) { return relativeOrAbsoluteModulePath, nil } return provider.packageId, nil diff --git a/docs/docs/cli-reference/package-init.md b/docs/docs/cli-reference/package-init.md new file mode 100644 index 0000000000..eab35e63cd --- /dev/null +++ b/docs/docs/cli-reference/package-init.md @@ -0,0 +1,17 @@ +# kurtosis package init +This command initializes the current directory to be a [Kurtosis package][package] by creating a [`kurtosis.yml` file][kurtosis-yml] with the given package name. + +``` +Usage: + kurtosis package init [flags] package_name +``` + +The `package_name` argument is the [locator][locators]to the package, in the format `github.com/USER/REPO`. + +This command accepts the following flags: +- `--main`: indicates that the created package is an [executable package][executable-package], and generates a `main.star` if one does not already exist. If a `main.star` already exists, does nothing. + +[package]: ../concepts-reference/packages.md +[kurtosis-yml]: ../concepts-reference/kurtosis-yml.md +[locators]: ../concepts-reference/locators.md +[executable-package]: ../concepts-reference/packages.md#runnable-packages \ No newline at end of file