From 672dfeb8cb6e871fd0155d47e1881cab0d5906b2 Mon Sep 17 00:00:00 2001 From: Soumil Paranjpay Date: Tue, 12 Dec 2023 17:33:13 +0530 Subject: [PATCH] final wasm changes Signed-off-by: Soumil Paranjpay --- transformer/external/wasmtransformer.go | 182 ++++++++++++++++++------ transformer/transformer.go | 1 + 2 files changed, 141 insertions(+), 42 deletions(-) diff --git a/transformer/external/wasmtransformer.go b/transformer/external/wasmtransformer.go index d559a04c3..83596b50d 100644 --- a/transformer/external/wasmtransformer.go +++ b/transformer/external/wasmtransformer.go @@ -20,10 +20,9 @@ import ( "encoding/binary" "encoding/json" "fmt" - "os" - "path/filepath" + "sort" + "strings" - "github.com/dchest/uniuri" "github.com/konveyor/move2kube/common" "github.com/konveyor/move2kube/environment" transformertypes "github.com/konveyor/move2kube/types/transformer" @@ -45,7 +44,6 @@ type WASM struct { Config transformertypes.Transformer Env *environment.Environment WASMConfig *WASMYamlConfig - VM *wasmedge.VM } // WASMYamlConfig is the format of wasm transformer yaml config @@ -54,37 +52,16 @@ type WASMYamlConfig struct { EnvList []core.EnvVar `yaml:"env,omitempty"` } -var ( - // WASMSharedDir is the directory where wasm transformer detect and transform output is stored - WASMSharedDir = "/var/tmp/m2k_detect_wasm_output" -) - // Init Initializes the transformer func (t *WASM) Init(tc transformertypes.Transformer, env *environment.Environment) (err error) { t.Config = tc + t.Env = env t.WASMConfig = &WASMYamlConfig{} if err := common.GetObjFromInterface(t.Config.Spec.Config, t.WASMConfig); err != nil { return fmt.Errorf("unable to load config for Transformer %+v into %T . Error: %q", t.Config.Spec.Config, t.WASMConfig, err) } - WASMSharedDir = filepath.Join(WASMSharedDir, uniuri.NewLen(5)) - os.MkdirAll(WASMSharedDir, common.DefaultDirectoryPermission) - // load wasm module - wasmedge.SetLogErrorLevel() - conf := wasmedge.NewConfigure(wasmedge.WASI) - vm := wasmedge.NewVMWithConfig(conf) - wasi := vm.GetImportModule(wasmedge.HostRegistration(wasmedge.WASI)) - wasi.InitWasi( - []string{}, - t.prepareEnv(), - []string{".:" + WASMSharedDir}, - ) - - err = vm.LoadWasmFile(t.WASMConfig.WASMModule) - vm.Validate() - vm.Instantiate() - t.VM = vm - return err + return nil } func (t *WASM) prepareEnv() []string { @@ -102,27 +79,68 @@ func (t *WASM) GetConfig() (transformertypes.Transformer, *environment.Environme // DirectoryDetect runs detect in each sub directory func (t *WASM) DirectoryDetect(dir string) (map[string][]transformertypes.Artifact, error) { - allocateResult, _ := t.VM.Execute("malloc", int32(len(dir)+1)) + wasmedge.SetLogErrorLevel() + conf := wasmedge.NewConfigure(wasmedge.WASI) + vm := wasmedge.NewVMWithConfig(conf) + + wasi := vm.GetImportModule(wasmedge.WASI) + wasi.InitWasi( + []string{}, + t.prepareEnv(), + []string{dir + ":" + dir}, + ) + + err := vm.LoadWasmFile(t.WASMConfig.WASMModule) + if err != nil { + return nil, err + } + err = vm.Validate() + if err != nil { + return nil, err + } + err = vm.Instantiate() + if err != nil { + return nil, err + } + _, err = vm.Execute("_start") + if err != nil { + return nil, err + } + + allocateResult, err := vm.Execute("malloc", int32(len(dir)+1)) + if err != nil { + return nil, err + } dirPointer := allocateResult[0].(int32) - mod := t.VM.GetActiveModule() + mod := vm.GetActiveModule() mem := mod.FindMemory("memory") - memData, _ := mem.GetData(uint(dirPointer), uint(len(dir)+1)) + memData, err := mem.GetData(uint(dirPointer), uint(len(dir)+1)) + if err != nil { + return nil, err + } copy(memData, dir) memData[len(dir)] = 0 - directoryDetectOutput, dderr := t.VM.Execute("directoryDetect", dirPointer) - var err error + directoryDetectOutput, dderr := vm.Execute("directoryDetect", dirPointer) if dderr != nil { err = fmt.Errorf("failed to execute directoryDetect in the wasm module. Error : %s", dderr.Error()) + return nil, err } directoryDetectOutputPointer := directoryDetectOutput[0].(int32) - memData, _ = mem.GetData(uint(directoryDetectOutputPointer), 8) + memData, err = mem.GetData(uint(directoryDetectOutputPointer), 8) + if err != nil { + return nil, err + } resultPointer := binary.LittleEndian.Uint32(memData[:4]) resultLength := binary.LittleEndian.Uint32(memData[4:]) - memData, _ = mem.GetData(uint(resultPointer), uint(resultLength)) + memData, err = mem.GetData(uint(resultPointer), uint(resultLength)) + if err != nil { + return nil, err + } logrus.Debug(string(memData)) + services := map[string][]transformertypes.Artifact{} // will handles errors - _ = json.Unmarshal(memData, &services) + err = json.Unmarshal(memData, &services) return services, err } @@ -133,21 +151,101 @@ func (t *WASM) Transform(newArtifacts []transformertypes.Artifact, alreadySeenAr data := make(map[string]interface{}) data["newArtifacts"] = newArtifacts data["oldArtifacts"] = alreadySeenArtifacts - dataByt, _ := json.Marshal(data) + dataByt, err := json.Marshal(data) + if err != nil { + return nil, nil, err + } dataStr := string(dataByt) - allocateResult, _ := t.VM.Execute("malloc", int32(len(dataStr)+1)) + + wasmedge.SetLogErrorLevel() + conf := wasmedge.NewConfigure(wasmedge.WASI) + vm := wasmedge.NewVMWithConfig(conf) + + wasi := vm.GetImportModule(wasmedge.WASI) + preopens := []string{} + for _, artifact := range newArtifacts { + for _, paths := range artifact.Paths { + for _, path := range paths { + preopens = append(preopens, path) + } + } + } + + sort.Slice(preopens, func(i, j int) bool { + l1, l2 := len(preopens[i]), len(preopens[j]) + if l1 != l2 { + return l1 < l2 + } + return preopens[i] < preopens[j] + }) + + deduplicatedPreopens := []string{} + for _, path := range preopens { + shouldSkip := false + for _, existingPath := range deduplicatedPreopens { + if strings.HasPrefix(path, existingPath) { + shouldSkip = true + break + } + } + + if !shouldSkip { + deduplicatedPreopens = append(deduplicatedPreopens, path) + } + } + + finalPreopens := []string{} + for _, path := range deduplicatedPreopens { + finalPreopens = append(finalPreopens, path+":"+path) + } + + wasi.InitWasi( + []string{}, + t.prepareEnv(), + finalPreopens, + ) + + err = vm.LoadWasmFile(t.WASMConfig.WASMModule) + if err != nil { + return nil, nil, err + } + err = vm.Validate() + if err != nil { + return nil, nil, err + } + err = vm.Instantiate() + if err != nil { + return nil, nil, err + } + _, err = vm.Execute("_start") + if err != nil { + return nil, nil, err + } + allocateResult, err := vm.Execute("malloc", int32(len(dataStr)+1)) + if err != nil { + return nil, nil, err + } dataPointer := allocateResult[0].(int32) - mod := t.VM.GetActiveModule() + mod := vm.GetActiveModule() mem := mod.FindMemory("memory") - memData, _ := mem.GetData(uint(dataPointer), uint(len(dataStr)+1)) + memData, err := mem.GetData(uint(dataPointer), uint(len(dataStr)+1)) + if err != nil { + return nil, nil, err + } copy(memData, dataStr) memData[len(dataStr)] = 0 - transformOutput, err := t.VM.Execute("transform", dataPointer) + transformOutput, err := vm.Execute("transform", dataPointer) transformOutputPointer := transformOutput[0].(int32) - memData, _ = mem.GetData(uint(transformOutputPointer), 8) + memData, err = mem.GetData(uint(transformOutputPointer), 8) + if err != nil { + return nil, nil, err + } resultPointer := binary.LittleEndian.Uint32(memData[:4]) resultLength := binary.LittleEndian.Uint32(memData[4:]) - memData, _ = mem.GetData(uint(resultPointer), uint(resultLength)) + memData, err = mem.GetData(uint(resultPointer), uint(resultLength)) + if err != nil { + return nil, nil, err + } logrus.Debug(string(memData)) var output transformertypes.TransformOutput json.Unmarshal(memData, &output) diff --git a/transformer/transformer.go b/transformer/transformer.go index 87b7aa1d6..058bc85ba 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -88,6 +88,7 @@ var ( func init() { transformerObjs := []Transformer{ + new(external.WASM), new(external.Starlark), new(external.Executable),