Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: kurtosis package init command #1547

Merged
merged 31 commits into from
Oct 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
1874996
local replace implemented in server and golang SDK
leoporoli Oct 9, 2023
b24410c
starting with ReplaceWithLocalAndThenWithRemote
leoporoli Oct 9, 2023
2575b4b
packageReplaceOptionsRepository added
leoporoli Oct 10, 2023
90435b7
Added TS library
leoporoli Oct 10, 2023
238f8e2
error msgs added
leoporoli Oct 11, 2023
fbe8ff2
Merge branch 'main' into lporoli/replace-wit-local-dependency
leoporoli Oct 11, 2023
0f84cc4
unused var removed
leoporoli Oct 11, 2023
a8838ac
fix
leoporoli Oct 11, 2023
d356715
Merge branch 'main' into lporoli/replace-wit-local-dependency
leoporoli Oct 11, 2023
3b06f22
Merge branch 'main' into lporoli/replace-wit-local-dependency
leoporoli Oct 11, 2023
7e71b75
starting with kurtosis package init cmd
leoporoli Oct 11, 2023
554dd64
Creating main.star file
leoporoli Oct 12, 2023
62d50b6
Merge branch 'main' into lporoli/ticket-1416
leoporoli Oct 12, 2023
4de321b
parsedGitUrl moved to golang API
leoporoli Oct 12, 2023
270e9fa
in progress
leoporoli Oct 12, 2023
dc84481
cmd added
leoporoli Oct 12, 2023
ce2460f
fix cmd package
leoporoli Oct 12, 2023
3eedd75
Merge branch 'main' into lporoli/ticket-1416
leoporoli Oct 12, 2023
dee12a1
Merge branch 'main' into lporoli/ticket-1416
leoporoli Oct 12, 2023
f711479
revert changes
leoporoli Oct 12, 2023
f374142
fix
leoporoli Oct 12, 2023
25c531a
docs added
leoporoli Oct 12, 2023
e5cb41d
main flag added
leoporoli Oct 12, 2023
2689ff5
Merge branch 'main' into lporoli/ticket-1416
leoporoli Oct 12, 2023
498c5aa
fix format
leoporoli Oct 12, 2023
b7c47bb
fix
leoporoli Oct 12, 2023
43bdc6d
remove unused function
leoporoli Oct 12, 2023
8cfa6e7
fix lint alerts
leoporoli Oct 13, 2023
401471a
Add new test
leoporoli Oct 17, 2023
65c6224
main.star file content with plan print instruction
leoporoli Oct 17, 2023
024baa2
Merge branch 'main' into lporoli/ticket-1416
leoporoli Oct 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions api/golang/core/lib/enclaves/kurtosis_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ type KurtosisYaml struct {
PackageReplaceOptions map[string]string `yaml:"replace"`
}

func NewKurtosisYaml(packageName string, packageDescription string, packageReplaceOptions map[string]string) *KurtosisYaml {
return &KurtosisYaml{PackageName: packageName, PackageDescription: packageDescription, PackageReplaceOptions: packageReplaceOptions}
}

func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) {
kurtosisYamlContents, err := os.ReadFile(kurtosisYamlFilepath)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
package git_package_content_provider
package shared_utils

import (
"fmt"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
"github.com/kurtosis-tech/stacktrace"
"net/url"
"path"
"strings"
)

const (
httpsSchema = "https"
urlPathSeparator = "/"
// for a valid GitURl we need it to look like github.com/author/moduleName
GithubDomainPrefix = "github.com"
httpsSchema = "https"
UrlPathSeparator = "/"
//MinimumSubPathsForValidGitURL for a valid GitURl we need it to look like github.com/author/moduleName
// the last two are the minimum requirements for a valid Startosis URL
minimumSubPathsForValidGitURL = 2
MinimumSubPathsForValidGitURL = 2

tagBranchOrCommitDelimiter = "@"
emptyTagBranchOrCommit = ""
Expand Down Expand Up @@ -54,44 +54,75 @@ func newParsedGitURL(moduleAuthor, moduleName, gitURL, relativeRepoPath, relativ
}
}

// parseGitURL this takes a Git url (GitHub) for now and converts it into the struct ParsedGitURL
func (parsedUrl *ParsedGitURL) GetModuleAuthor() string {
return parsedUrl.moduleAuthor
}

func (parsedUrl *ParsedGitURL) GetModuleName() string {
return parsedUrl.moduleName
}

func (parsedUrl *ParsedGitURL) GetGitURL() string {
return parsedUrl.gitURL
}

func (parsedUrl *ParsedGitURL) GetRelativeRepoPath() string {
return parsedUrl.relativeRepoPath
}

func (parsedUrl *ParsedGitURL) GetRelativeFilePath() string {
return parsedUrl.relativeFilePath
}

func (parsedUrl *ParsedGitURL) GetTagBranchOrCommit() string {
return parsedUrl.tagBranchOrCommit
}

func (parsedUrl *ParsedGitURL) GetAbsoluteLocatorRelativeToThisURL(relativeUrl string) string {
if strings.HasPrefix(relativeUrl, packageRootPrefixIndicatorInRelativeLocators) {
return path.Join(GithubDomainPrefix, parsedUrl.relativeRepoPath, relativeUrl)
}
return path.Join(GithubDomainPrefix, path.Dir(parsedUrl.relativeFilePath), relativeUrl)
}

// ParseGitURL this takes a Git url (GitHub) for now and converts it into the struct ParsedGitURL
// This can in the future be extended to GitLab or BitBucket or any other Git Host
func parseGitURL(packageURL string) (*ParsedGitURL, *startosis_errors.InterpretationError) {
func ParseGitURL(packageURL string) (*ParsedGitURL, error) {
// we expect something like github.com/author/module/path.star
// we don't want schemas
parsedURL, err := url.Parse(packageURL)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Error parsing the URL of module '%v'", packageURL)
return nil, stacktrace.Propagate(err, "Error parsing the URL of module '%v'", packageURL)
}
if parsedURL.Scheme != "" {
return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module '%v'. Expected schema to be empty got '%v'", packageURL, parsedURL.Scheme)
return nil, stacktrace.NewError("Error parsing the URL of module '%v'. Expected schema to be empty got '%v'", packageURL, parsedURL.Scheme)
}

// we prefix schema and make sure that the URL still parses
packageURLPrefixedWithHttps := httpsSchema + "://" + packageURL
parsedURL, err = url.Parse(packageURLPrefixedWithHttps)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Error parsing the URL with scheme for module '%v'", packageURLPrefixedWithHttps)
return nil, stacktrace.Propagate(err, "Error parsing the URL with scheme for module '%v'", packageURLPrefixedWithHttps)
}
if parsedURL.Host != startosis_constants.GithubDomainPrefix {
return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module. We only support modules on Github for now but got '%v'", packageURL)
if parsedURL.Host != GithubDomainPrefix {
return nil, stacktrace.NewError("Error parsing the URL of module. We only support modules on Github for now but got '%v'", packageURL)
}

pathWithoutVersion, maybeTagBranchOrCommit := parseOutTagBranchOrCommit(parsedURL.Path)

splitURLPath := cleanPathAndSplit(pathWithoutVersion)

if len(splitURLPath) < minimumSubPathsForValidGitURL {
return nil, startosis_errors.NewInterpretationError("Error parsing the URL of module: '%v'. The path should contain at least %d subpaths got '%v'", packageURL, minimumSubPathsForValidGitURL, splitURLPath)
if len(splitURLPath) < MinimumSubPathsForValidGitURL {
return nil, stacktrace.NewError("Error parsing the URL of module: '%v'. The path should contain at least %d subpaths got '%v'", packageURL, MinimumSubPathsForValidGitURL, splitURLPath)
}

moduleAuthor := splitURLPath[0]
moduleName := splitURLPath[1]
gitURL := fmt.Sprintf("%v://%v/%v/%v.git", httpsSchema, startosis_constants.GithubDomainPrefix, moduleAuthor, moduleName)
gitURL := fmt.Sprintf("%v://%v/%v/%v.git", httpsSchema, GithubDomainPrefix, moduleAuthor, moduleName)
relativeModulePath := path.Join(moduleAuthor, moduleName)

relativeFilePath := ""
if len(splitURLPath) > minimumSubPathsForValidGitURL {
if len(splitURLPath) > MinimumSubPathsForValidGitURL {
relativeFilePath = path.Join(splitURLPath...)
}

Expand All @@ -107,17 +138,10 @@ func parseGitURL(packageURL string) (*ParsedGitURL, *startosis_errors.Interpreta
return parsedGitURL, nil
}

func (parsedUrl *ParsedGitURL) getAbsoluteLocatorRelativeToThisURL(relativeUrl string) string {
if strings.HasPrefix(relativeUrl, packageRootPrefixIndicatorInRelativeLocators) {
return path.Join(startosis_constants.GithubDomainPrefix, parsedUrl.relativeRepoPath, relativeUrl)
}
return path.Join(startosis_constants.GithubDomainPrefix, path.Dir(parsedUrl.relativeFilePath), relativeUrl)
}

// cleanPath removes empty "" from the string slice
func cleanPathAndSplit(urlPath string) []string {
cleanPath := path.Clean(urlPath)
splitPath := strings.Split(cleanPath, urlPathSeparator)
splitPath := strings.Split(cleanPath, UrlPathSeparator)
var sliceWithoutEmptyStrings []string
for _, subPath := range splitPath {
if subPath != "" {
Expand All @@ -138,7 +162,7 @@ func parseOutTagBranchOrCommit(input string) (string, string) {
// 3- github.com/kurtosis-tech/sample-dependency-package/main.star@foo/bar - here the tag is foo/bar;
// 4- github.com/kurtosis-tech/sample-dependency-package@foo/bar/mains.tar - here the tag is foo/bar; while file is /kurtosis-tech/sample-dependency-package/main.star
// we check if there is a file in maybeTagBranchOrCommitWithFile and then add it to pathWithoutVersion
maybeTagBranchOrCommit, lastSectionOfTagBranchCommitWithFile, _ := cutLast(maybeTagBranchOrCommitWithFile, urlPathSeparator)
maybeTagBranchOrCommit, lastSectionOfTagBranchCommitWithFile, _ := cutLast(maybeTagBranchOrCommitWithFile, UrlPathSeparator)

if lastSectionOfTagBranchCommitWithFile != "" && strings.Contains(lastSectionOfTagBranchCommitWithFile, extensionCharacter) {
// we assume pathWithoutVersion does not contain a file inside yet
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package git_package_content_provider
package shared_utils

import (
"fmt"
Expand All @@ -18,7 +18,7 @@ const (
)

func TestParsedGitURL_SimpleParse(t *testing.T) {
parsedURL, err := parseGitURL(githubSampleURL)
parsedURL, err := ParseGitURL(githubSampleURL)
require.Nil(t, err)

expectedParsedURL := newParsedGitURL(
Expand All @@ -35,7 +35,7 @@ func TestParsedGitURL_SimpleParse(t *testing.T) {

func TestParsedGitURL_FailsOnNonGithubURL(t *testing.T) {
nonGithubURL := "kurtosis-git.com/" + testModuleAuthor + "/" + testModuleName + "/" + testFileName
_, err := parseGitURL(nonGithubURL)
_, err := ParseGitURL(nonGithubURL)
require.NotNil(t, err)

expectedErrorMsg := "We only support modules on Github for now"
Expand All @@ -46,7 +46,7 @@ func TestParsedGitURL_FailsOnNonGithubURL(t *testing.T) {
func TestParsedGitURL_FailsOnNonNonEmptySchema(t *testing.T) {
ftpSchema := "ftp"
nonGithubURL := ftpSchema + "://github.com/" + testModuleAuthor + "/" + testModuleName + "/" + testFileName
_, err := parseGitURL(nonGithubURL)
_, err := ParseGitURL(nonGithubURL)
require.NotNil(t, err)

expectedErrorMsg := fmt.Sprintf("Expected schema to be empty got '%v'", ftpSchema)
Expand All @@ -56,19 +56,19 @@ func TestParsedGitURL_FailsOnNonNonEmptySchema(t *testing.T) {

func TestParsedGitURL_IfNoFileThenRelativeFilePathIsEmpty(t *testing.T) {
pathWithoutFile := "github.com/" + testModuleAuthor + "/" + testModuleName
parsedURL, err := parseGitURL(pathWithoutFile)
parsedURL, err := ParseGitURL(pathWithoutFile)
require.Nil(t, err)
require.Equal(t, "", parsedURL.relativeFilePath)
}

func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) {
escapedURLWithoutStartosisFile := "github.com/../etc/passwd"
parsedURL, err := parseGitURL(escapedURLWithoutStartosisFile)
parsedURL, err := ParseGitURL(escapedURLWithoutStartosisFile)
require.Nil(t, err)
require.Equal(t, "", parsedURL.relativeFilePath)

escapedURLWithStartosisFile := "github.com/../../etc/passwd/startosis.star"
parsedURL, err = parseGitURL(escapedURLWithStartosisFile)
parsedURL, err = ParseGitURL(escapedURLWithStartosisFile)
require.Nil(t, err)
require.Equal(t, parsedURL.moduleAuthor, "etc")
require.Equal(t, parsedURL.moduleName, "passwd")
Expand All @@ -77,7 +77,7 @@ func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) {
require.Equal(t, parsedURL.relativeRepoPath, "etc/passwd")

escapedURLWithStartosisFile = "github.com/foo/../etc/passwd/startosis.star"
parsedURL, err = parseGitURL(escapedURLWithStartosisFile)
parsedURL, err = ParseGitURL(escapedURLWithStartosisFile)
require.Nil(t, err)
require.Equal(t, parsedURL.moduleAuthor, "etc")
require.Equal(t, parsedURL.moduleName, "passwd")
Expand All @@ -86,14 +86,14 @@ func TestParsedGitURL_ParsingGetsRidOfAnyPathEscapes(t *testing.T) {
require.Equal(t, parsedURL.relativeRepoPath, "etc/passwd")

escapedURLWithStartosisFile = "github.com/foo/../etc/../passwd"
_, err = parseGitURL(escapedURLWithStartosisFile)
_, err = ParseGitURL(escapedURLWithStartosisFile)
require.NotNil(t, err)
expectedErrorMsg := fmt.Sprintf("Error parsing the URL of module: '%s'. The path should contain at least 2 subpaths got '[passwd]'", escapedURLWithStartosisFile)
require.Contains(t, err.Error(), expectedErrorMsg)
}

func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) {
parsedURL, err := parseGitURL(githubSampleUrlWithTag)
parsedURL, err := ParseGitURL(githubSampleUrlWithTag)
require.Nil(t, err)

expectedParsedURL := newParsedGitURL(
Expand All @@ -107,7 +107,7 @@ func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) {

require.Equal(t, expectedParsedURL, parsedURL)

parsedURL, err = parseGitURL(githubSampleUrlWithBranchContainingVersioningDelimiter)
parsedURL, err = ParseGitURL(githubSampleUrlWithBranchContainingVersioningDelimiter)
require.Nil(t, err)

expectedParsedURL = newParsedGitURL(
Expand All @@ -123,46 +123,46 @@ func TestParsedGitURL_WorksWithVersioningInformation(t *testing.T) {
}

func TestParsedGitUrl_ResolvesRelativeUrl(t *testing.T) {
parsedUrl, err := parseGitURL(githubSampleURL)
parsedUrl, err := ParseGitURL(githubSampleURL)
require.Nil(t, err)

relativeUrl := "./lib.star"
absoluteUrl := parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl)
absoluteUrl := parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl)
require.Nil(t, err)
expected := "github.com/kurtosis-tech/sample-startosis-load/lib.star"
require.Equal(t, expected, absoluteUrl)

relativeUrl = "./src/lib.star"
absoluteUrl = parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl)
absoluteUrl = parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl)
require.Nil(t, err)
expected = "github.com/kurtosis-tech/sample-startosis-load/src/lib.star"
require.Equal(t, expected, absoluteUrl)
}

func TestParsedGitUrl_ResolvesRelativeUrlForUrlWithTag(t *testing.T) {
parsedUrl, err := parseGitURL(githubSampleUrlWithTag)
parsedUrl, err := ParseGitURL(githubSampleUrlWithTag)
require.Nil(t, err)

relativeUrl := "./lib.star"
absoluteUrl := parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl)
absoluteUrl := parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl)
require.Nil(t, err)
expected := "github.com/kurtosis-tech/sample-startosis-load/lib.star"
require.Equal(t, expected, absoluteUrl)

relativeUrl = "./src/lib.star"
absoluteUrl = parsedUrl.getAbsoluteLocatorRelativeToThisURL(relativeUrl)
absoluteUrl = parsedUrl.GetAbsoluteLocatorRelativeToThisURL(relativeUrl)
require.Nil(t, err)
expected = "github.com/kurtosis-tech/sample-startosis-load/src/lib.star"
require.Equal(t, expected, absoluteUrl)
}

func TestParsedGitUrl_ResolvesWithUrlWithVersionBranchWithSlash(t *testing.T) {
parsedUrl, err := parseGitURL(githubSampleUrlWithVersionWithSlash)
parsedUrl, err := ParseGitURL(githubSampleUrlWithVersionWithSlash)
require.Nil(t, err)

require.Equal(t, "foo/bar", parsedUrl.tagBranchOrCommit)

parsedUrl, err = parseGitURL(githubSampleUrlWithVersionWithSlashAndFile)
parsedUrl, err = ParseGitURL(githubSampleUrlWithVersionWithSlashAndFile)
require.Nil(t, err)
require.Equal(t, "foo/bar", parsedUrl.tagBranchOrCommit)
require.Equal(t, "kurtosis-tech/sample-startosis-load/main.star", parsedUrl.relativeFilePath)
Expand Down
2 changes: 2 additions & 0 deletions cli/cli/command_str_consts/command_str_consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ const (
VersionCmdStr = "version"
ImportCmdStr = "import"
GatewayCmdStr = "gateway"
PackageCmdStr = "package"
InitCmdStr = "init"
PortCmdStr = "port"
PortPrintCmdStr = "print"
WebCmdStr = "web"
Expand Down
85 changes: 85 additions & 0 deletions cli/cli/commands/package/init_cmd/init_cmd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package init_cmd

import (
"context"
"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils"
"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel"
"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args"
"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_package"
"github.com/kurtosis-tech/stacktrace"
"os"
)

const (
packageNameArgKey = "package-name"
packageNameArgDefaultValue = ""
packageNameArgIsOptional = false
packageNameArgIsGreedy = false

executablePackageFlagKey = "main"
executablePackageFlagDefaultValue = "false"
)

var InitCmd = &lowlevel.LowlevelKurtosisCommand{
CommandStr: command_str_consts.InitCmdStr,
ShortDescription: "Creates a new Kurtosis package",
LongDescription: "This command initializes the current directory to be a Kurtosis package by creating a `kurtosis.yml` with the given package name.",
Args: []*args.ArgConfig{
{
Key: packageNameArgKey,
DefaultValue: packageNameArgDefaultValue,
IsOptional: packageNameArgIsOptional,
IsGreedy: packageNameArgIsGreedy,
ValidationFunc: validatePackageNameArg,
},
},
Flags: []*flags.FlagConfig{
{
Key: executablePackageFlagKey,
Usage: "indicates that the created package is an executable package, and generates a 'main.star' if one does not already exist.",
Type: flags.FlagType_Bool,
Default: executablePackageFlagDefaultValue,
},
},
PreValidationAndRunFunc: nil,
RunFunc: run,
PostValidationAndRunFunc: nil,
}

func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) error {
packageNameArg, err := args.GetNonGreedyArg(packageNameArgKey)
if err != nil {
return stacktrace.Propagate(err, "an error occurred getting the value of argument with key '%v'", packageNameArgKey)
}

executablePackageFlag, err := flags.GetBool(executablePackageFlagKey)
if err != nil {
return stacktrace.Propagate(err, "an error occurred getting the value of flag '%v'", executablePackageFlagKey)
}

packageDestinationDirpath, err := os.Getwd()
if err != nil {
return stacktrace.Propagate(err, "An error occurred getting the current working directory for creating the Kurtosis package")
}

if err := kurtosis_package.InitializeKurtosisPackage(packageDestinationDirpath, packageNameArg, executablePackageFlag); err != nil {
return stacktrace.Propagate(err, "An error occurred initializing the Kurtosis package '%s' in '%s'", packageNameArg, packageDestinationDirpath)
}

return nil
}

func validatePackageNameArg(_ context.Context, _ *flags.ParsedFlags, args *args.ParsedArgs) error {
packageNameArg, err := args.GetNonGreedyArg(packageNameArgKey)
if err != nil {
return stacktrace.Propagate(err, "an error occurred getting the value of argument with key '%v'", packageNameArgKey)
}

if _, err := shared_utils.ParseGitURL(packageNameArg); err != nil {
return stacktrace.Propagate(err, "An erro occurred validating package name '%v', invalid GitHub URL", packageNameArg)
}

return nil
}
Loading