Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: podman attempt v2 #2518

Open
wants to merge 55 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
59ba36d
fix: add enclave-manager/local to workspace
marijanp May 25, 2024
bc0e31c
feat: package kurtosis with nix
marijanp May 25, 2024
41bd2dc
feat: ignore nix related files
marijanp May 25, 2024
744e330
use podman socket
tedim52 Jul 12, 2024
70fa36a
turn off log collector
tedim52 Jul 12, 2024
df42701
fix volume inconsistency match
tedim52 Jul 12, 2024
68c1387
retrieve network id
tedim52 Jul 12, 2024
e062bf1
change host ip
tedim52 Jul 12, 2024
15aa10a
add xdg fallback
tedim52 Jul 12, 2024
555fcb3
use local goreleaser
tedim52 Jul 12, 2024
ca0da34
Merge branch 'nixify' into tedi/podman
tedim52 Jul 12, 2024
3e20f4c
Revert "use podman socket"
tedim52 Jul 12, 2024
f54369b
fix get engine logs and components
tedim52 Jul 12, 2024
e48fc23
bridge -> podman
tedim52 Jul 12, 2024
9789f1c
disable security opts
tedim52 Jul 25, 2024
4edfe23
use podman for image building
tedim52 Jul 25, 2024
aaca583
fix
tedim52 Jul 25, 2024
900a531
wait and list networks before connecting
tedim52 Jul 31, 2024
572f4b8
improve err logs for storing pkg contents
tedim52 Aug 1, 2024
aba722a
start dependency list
tedim52 Aug 23, 2024
507ef13
add images and package dependencies to plan yaml
tedim52 Aug 30, 2024
546c1c9
hook up to cli
tedim52 Aug 30, 2024
8df95e7
add get starlark plan yaml to api
tedim52 Aug 30, 2024
d0bba73
update tests
tedim52 Aug 30, 2024
3831674
make imgs unique and sort
tedim52 Aug 30, 2024
725cf84
Merge branch 'main' into tedi/dependencylist
tedim52 Aug 30, 2024
65da4ab
refactor plan yaml
tedim52 Aug 30, 2024
b712fe8
only add module prefix and dont add parent package
tedim52 Aug 30, 2024
c1cd1fc
maybe parse yaml
tedim52 Aug 30, 2024
554b754
pull dependencies locally
tedim52 Sep 11, 2024
18c2591
fix cloning pkgs
tedim52 Sep 12, 2024
7e56e5b
replace magic vlues
tedim52 Sep 12, 2024
24bbebb
exhaust struct
tedim52 Sep 12, 2024
2422cf2
Merge branch 'tedi/dependencylist' into tedi/podmansock
tedim52 Sep 12, 2024
09e90ee
Merge branch 'main' into tedi/dependencylist
tedim52 Oct 17, 2024
070fa60
Merge branch 'main' into tedi/dependencylist
tedim52 Nov 14, 2024
f5d4014
start addressing comments
tedim52 Nov 14, 2024
db66969
remove dependencies only flag
tedim52 Nov 15, 2024
ec6e9af
rename test file
tedim52 Nov 20, 2024
5121860
typo
tedim52 Nov 20, 2024
294c325
remove add images
tedim52 Nov 20, 2024
6b489ca
clarify get module prefix
tedim52 Nov 20, 2024
807de9b
finish addressing comments
tedim52 Nov 20, 2024
0a0c10b
account for official imgs
tedim52 Nov 20, 2024
f7e5d9d
regen proto to remove dependencies only
tedim52 Nov 20, 2024
3de186f
change location of err handle
tedim52 Nov 20, 2024
a938a40
use work dir for kurt yml check
tedim52 Nov 20, 2024
57ab266
remove extra colon
tedim52 Nov 20, 2024
c2ce7f4
lint
tedim52 Nov 20, 2024
dcb43e1
change docker auth err to warn logs
tedim52 Nov 20, 2024
8f763bd
Merge branch 'main' into tedi/dependencylist
tedim52 Nov 20, 2024
d9e4235
Merge branch 'tedi/dependencylist' into tedi/podmansock
tedim52 Nov 21, 2024
18de25e
pull local package
tedim52 Nov 25, 2024
9820014
undo breaking change
tedim52 Nov 25, 2024
1587f41
Merge branch 'tedi/dependencylist' into tedi/podmansock
tedim52 Nov 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -131,4 +131,8 @@ lerna-debug.log
# contains several libraries that get shipped
yarn.lock
package.json
package-lock.json
package-lock.json

# Nix
# ignores the default result symlink created when building with nix
result

Large diffs are not rendered by default.

69 changes: 69 additions & 0 deletions api/golang/core/lib/enclaves/enclave_context.go
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,75 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis
return response, nil
}

func (enclaveCtx *EnclaveContext) GetStarlarkRemotePackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) {
serializedParams, err := maybeParseYaml(serializedParams)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageId, serializedParams)
}
response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{
PackageId: packageId,
SerializedParams: &serializedParams,
IsRemote: true,
RelativePathToMainFile: nil,
MainFunctionName: nil,
})
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the Starlark package plan yaml.")
}
return response, nil
}

func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageRootPath string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) {
packageName, packageReplaceOptions, err := getPackageNameAndReplaceOptions(packageRootPath)
if err != nil {
return nil, err
}

serializedParams, err = maybeParseYaml(serializedParams)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageName, serializedParams)
}

err = enclaveCtx.uploadStarlarkPackage(packageName, packageRootPath)
if err != nil {
return nil, stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", packageRootPath)
}

if len(packageReplaceOptions) > 0 {
if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, packageReplaceOptions); err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", packageReplaceOptions)
}
}

response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{
PackageId: packageName,
SerializedParams: &serializedParams,
IsRemote: false,
RelativePathToMainFile: nil,
MainFunctionName: nil,
})
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the Starlark package plan yaml.")
}
return response, nil
}

func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) {
serializedParams, err := maybeParseYaml(serializedParams)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams)
}
response, err := enclaveCtx.client.GetStarlarkScriptPlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkScriptPlanYamlArgs{
SerializedScript: serializedScript,
SerializedParams: &serializedParams,
MainFunctionName: nil,
})
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark script plan yaml.")
}
return response, nil
}

// ====================================================================================================
//
// Private helper methods
Expand Down
7 changes: 5 additions & 2 deletions api/protobuf/core/api_container_service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -607,9 +607,12 @@ message StarlarkPackagePlanYamlArgs {
// This should be a valid JSON string
optional string serialized_params = 2;

// whether or not this is package yaml should be pulled from on disk package or cloned
bool is_remote = 3;

// The relative main file filepath, the default value is the "main.star" file in the root of a package
optional string relative_path_to_main_file = 3;
optional string relative_path_to_main_file = 4;

// The name of the main function, the default value is "run"
optional string main_function_name = 4;
optional string main_function_name = 5;
}
7 changes: 5 additions & 2 deletions api/rust/src/api_container_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -719,13 +719,16 @@ pub struct StarlarkPackagePlanYamlArgs {
/// This should be a valid JSON string
#[prost(string, optional, tag = "2")]
pub serialized_params: ::core::option::Option<::prost::alloc::string::String>,
/// whether or not this is package yaml should be pulled from on disk package or cloned
#[prost(bool, tag = "3")]
pub is_remote: bool,
/// The relative main file filepath, the default value is the "main.star" file in the root of a package
#[prost(string, optional, tag = "3")]
#[prost(string, optional, tag = "4")]
pub relative_path_to_main_file: ::core::option::Option<
::prost::alloc::string::String,
>,
/// The name of the main function, the default value is "run"
#[prost(string, optional, tag = "4")]
#[prost(string, optional, tag = "5")]
pub main_function_name: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1546,6 +1546,9 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message {
hasSerializedParams(): boolean;
clearSerializedParams(): StarlarkPackagePlanYamlArgs;

getIsRemote(): boolean;
setIsRemote(value: boolean): StarlarkPackagePlanYamlArgs;

getRelativePathToMainFile(): string;
setRelativePathToMainFile(value: string): StarlarkPackagePlanYamlArgs;
hasRelativePathToMainFile(): boolean;
Expand All @@ -1568,6 +1571,7 @@ export namespace StarlarkPackagePlanYamlArgs {
export type AsObject = {
packageId: string,
serializedParams?: string,
isRemote: boolean,
relativePathToMainFile?: string,
mainFunctionName?: string,
}
Expand All @@ -1579,12 +1583,12 @@ export namespace StarlarkPackagePlanYamlArgs {

export enum RelativePathToMainFileCase {
_RELATIVE_PATH_TO_MAIN_FILE_NOT_SET = 0,
RELATIVE_PATH_TO_MAIN_FILE = 3,
RELATIVE_PATH_TO_MAIN_FILE = 4,
}

export enum MainFunctionNameCase {
_MAIN_FUNCTION_NAME_NOT_SET = 0,
MAIN_FUNCTION_NAME = 4,
MAIN_FUNCTION_NAME = 5,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11108,8 +11108,9 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeI
var f, obj = {
packageId: jspb.Message.getFieldWithDefault(msg, 1, ""),
serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""),
relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 3, ""),
mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, "")
isRemote: jspb.Message.getBooleanFieldWithDefault(msg, 3, false),
relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 4, ""),
mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "")
};

if (includeInstance) {
Expand Down Expand Up @@ -11155,10 +11156,14 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader
msg.setSerializedParams(value);
break;
case 3:
var value = /** @type {boolean} */ (reader.readBool());
msg.setIsRemote(value);
break;
case 4:
var value = /** @type {string} */ (reader.readString());
msg.setRelativePathToMainFile(value);
break;
case 4:
case 5:
var value = /** @type {string} */ (reader.readString());
msg.setMainFunctionName(value);
break;
Expand Down Expand Up @@ -11205,9 +11210,9 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu
f
);
}
f = /** @type {string} */ (jspb.Message.getField(message, 3));
if (f != null) {
writer.writeString(
f = message.getIsRemote();
if (f) {
writer.writeBool(
3,
f
);
Expand All @@ -11219,6 +11224,13 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu
f
);
}
f = /** @type {string} */ (jspb.Message.getField(message, 5));
if (f != null) {
writer.writeString(
5,
f
);
}
};


Expand Down Expand Up @@ -11277,11 +11289,29 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasSerializedParam


/**
* optional string relative_path_to_main_file = 3;
* optional bool is_remote = 3;
* @return {boolean}
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getIsRemote = function() {
return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false));
};


/**
* @param {boolean} value
* @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setIsRemote = function(value) {
return jspb.Message.setProto3BooleanField(this, 3, value);
};


/**
* optional string relative_path_to_main_file = 4;
* @return {string}
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getRelativePathToMainFile = function() {
return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, ""));
return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, ""));
};


Expand All @@ -11290,7 +11320,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getRelativePathToM
* @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setRelativePathToMainFile = function(value) {
return jspb.Message.setField(this, 3, value);
return jspb.Message.setField(this, 4, value);
};


Expand All @@ -11299,7 +11329,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setRelativePathToM
* @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearRelativePathToMainFile = function() {
return jspb.Message.setField(this, 3, undefined);
return jspb.Message.setField(this, 4, undefined);
};


Expand All @@ -11308,16 +11338,16 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearRelativePathT
* @return {boolean}
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasRelativePathToMainFile = function() {
return jspb.Message.getField(this, 3) != null;
return jspb.Message.getField(this, 4) != null;
};


/**
* optional string main_function_name = 4;
* optional string main_function_name = 5;
* @return {string}
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getMainFunctionName = function() {
return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, ""));
return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 5, ""));
};


Expand All @@ -11326,7 +11356,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getMainFunctionNam
* @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setMainFunctionName = function(value) {
return jspb.Message.setField(this, 4, value);
return jspb.Message.setField(this, 5, value);
};


Expand All @@ -11335,7 +11365,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setMainFunctionNam
* @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearMainFunctionName = function() {
return jspb.Message.setField(this, 4, undefined);
return jspb.Message.setField(this, 5, undefined);
};


Expand All @@ -11344,7 +11374,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearMainFunctionN
* @return {boolean}
*/
proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasMainFunctionName = function() {
return jspb.Message.getField(this, 4) != null;
return jspb.Message.getField(this, 5) != null;
};


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1914,17 +1914,24 @@ export declare class StarlarkPackagePlanYamlArgs extends Message<StarlarkPackage
*/
serializedParams?: string;

/**
* whether or not this is package yaml should be pulled from on disk package or cloned
*
* @generated from field: bool is_remote = 3;
*/
isRemote: boolean;

/**
* The relative main file filepath, the default value is the "main.star" file in the root of a package
*
* @generated from field: optional string relative_path_to_main_file = 3;
* @generated from field: optional string relative_path_to_main_file = 4;
*/
relativePathToMainFile?: string;

/**
* The name of the main function, the default value is "run"
*
* @generated from field: optional string main_function_name = 4;
* @generated from field: optional string main_function_name = 5;
*/
mainFunctionName?: string;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -670,8 +670,9 @@ export const StarlarkPackagePlanYamlArgs = proto3.makeMessageType(
() => [
{ no: 1, name: "package_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 3, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 4, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 3, name: "is_remote", kind: "scalar", T: 8 /* ScalarType.BOOL */ },
{ no: 4, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
{ no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
],
);

Loading
Loading