diff --git a/go.mod b/go.mod index d845d1d..6a428b9 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,16 @@ require ( github.com/ProtonMail/gopenpgp/v2 v2.7.5 github.com/bwmarrin/snowflake v0.3.0 github.com/cidverse/cidverseutils/ci v0.0.0-20240501195623-588d642a5813 - github.com/cidverse/cidverseutils/compress v0.0.0-20240501195623-588d642a5813 + github.com/cidverse/cidverseutils/compress v0.0.0-20240507202753-435d763dc50c github.com/cidverse/cidverseutils/containerruntime v0.0.0-20240501195623-588d642a5813 - github.com/cidverse/cidverseutils/filesystem v0.0.0-20240501195623-588d642a5813 - github.com/cidverse/cidverseutils/hash v0.0.0-20240501195623-588d642a5813 + github.com/cidverse/cidverseutils/filesystem v0.0.0-20240507203508-adb806ff2095 + github.com/cidverse/cidverseutils/hash v0.0.0-20240507202753-435d763dc50c github.com/cidverse/cidverseutils/network v0.0.0-20240501195623-588d642a5813 + github.com/cidverse/cidverseutils/redact v0.0.0-20240507202753-435d763dc50c github.com/cidverse/cidverseutils/version v0.0.0-20240501195623-588d642a5813 github.com/cidverse/go-rules v0.0.0-20231112122021-075e5e6f8abc github.com/cidverse/go-vcs v0.0.0-20240402102656-5c7ce4c133c6 - github.com/cidverse/normalizeci v1.1.1-0.20240323134319-1eade1c37ec8 + github.com/cidverse/normalizeci v1.1.1-0.20240507162324-423657017129 github.com/cidverse/repoanalyzer v0.1.0 github.com/go-resty/resty/v2 v2.12.0 github.com/google/uuid v1.6.0 @@ -47,7 +48,7 @@ require ( github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/charlievieth/fastwalk v1.0.3 // indirect - github.com/cidverse/cidverseutils/exec v0.0.0-20240501195623-588d642a5813 // indirect + github.com/cidverse/cidverseutils/exec v0.0.0-20240507202753-435d763dc50c // indirect github.com/cloudflare/circl v1.3.8 // indirect github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -62,7 +63,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/cel-go v0.20.1 // indirect - github.com/google/go-github/v60 v60.0.0 // indirect + github.com/google/go-github/v61 v61.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gosimple/slug v1.14.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect diff --git a/go.sum b/go.sum index ebf5aa1..285920a 100644 --- a/go.sum +++ b/go.sum @@ -28,26 +28,28 @@ github.com/charlievieth/fastwalk v1.0.3 h1:eNWFaNPe5srPqQ5yyDbhAf11paeZaHWcihRhp github.com/charlievieth/fastwalk v1.0.3/go.mod h1:JSfglY/gmL/rqsUS1NCsJTocB5n6sSl9ApAqif4CUbs= github.com/cidverse/cidverseutils/ci v0.0.0-20240501195623-588d642a5813 h1:qcvZGs2HJb+4RTH9oPEeawqpYGXP0Q4W7MUkpG8PRQs= github.com/cidverse/cidverseutils/ci v0.0.0-20240501195623-588d642a5813/go.mod h1:uOMGB2Kh/0HA936tlnzgBvrxv4hmXYeSbJxP383nHKU= -github.com/cidverse/cidverseutils/compress v0.0.0-20240501195623-588d642a5813 h1:o+046dcOA15Kiw+p+N6atZAUj3IWxHLilhuR2f+34Qw= -github.com/cidverse/cidverseutils/compress v0.0.0-20240501195623-588d642a5813/go.mod h1:Xga0FDD+hHizd9zH99+DrbqXTAsblYQVEolYiOCN/Ug= +github.com/cidverse/cidverseutils/compress v0.0.0-20240507202753-435d763dc50c h1:PXSJvHUYGabSee2uq/OheOwTNptyhM7Msk8TRRZtGy4= +github.com/cidverse/cidverseutils/compress v0.0.0-20240507202753-435d763dc50c/go.mod h1:Xga0FDD+hHizd9zH99+DrbqXTAsblYQVEolYiOCN/Ug= github.com/cidverse/cidverseutils/containerruntime v0.0.0-20240501195623-588d642a5813 h1:tFrX9r+rHVrr7DfAfUJEC+BWjPXy/MnfmKvJ0klZBuE= github.com/cidverse/cidverseutils/containerruntime v0.0.0-20240501195623-588d642a5813/go.mod h1:749tfa5AB/Z7apXXT9ttr/rvSBdae1xo2DZAVRB9a7k= -github.com/cidverse/cidverseutils/exec v0.0.0-20240501195623-588d642a5813 h1:mDZDNTUBNuRlRS/TOJaz/cUsVikAsoOMR9OS3JXd0L4= -github.com/cidverse/cidverseutils/exec v0.0.0-20240501195623-588d642a5813/go.mod h1:DHW1LndctiozzTrK8lgBtVFjdc3ynZviR5nEACmnB5E= -github.com/cidverse/cidverseutils/filesystem v0.0.0-20240501195623-588d642a5813 h1:6wNwyccNzwxlK3dA9dPtaXLqx3mNjzdjolKIcpJf/Ls= -github.com/cidverse/cidverseutils/filesystem v0.0.0-20240501195623-588d642a5813/go.mod h1:rw4oLD+q/7HCMkQUooxVz4He06ORyoAn2cXLFqpBinQ= -github.com/cidverse/cidverseutils/hash v0.0.0-20240501195623-588d642a5813 h1:caRJx02nE+NDSoKLBnrxFISMV+D4xYadDwALvM8BGvc= -github.com/cidverse/cidverseutils/hash v0.0.0-20240501195623-588d642a5813/go.mod h1:MiRf2odL+20YA8iFNU14BT3o5d/1IOku+yjMYCxzXxo= +github.com/cidverse/cidverseutils/exec v0.0.0-20240507202753-435d763dc50c h1:GOiHymOXDcOVBYpvbr7m8QMB9OV+0PCzapKgcp9rKoU= +github.com/cidverse/cidverseutils/exec v0.0.0-20240507202753-435d763dc50c/go.mod h1:DHW1LndctiozzTrK8lgBtVFjdc3ynZviR5nEACmnB5E= +github.com/cidverse/cidverseutils/filesystem v0.0.0-20240507203508-adb806ff2095 h1:yZzje9c207/OgHS2JLkF6A5pDKtKZt8aZct21zJ8/iU= +github.com/cidverse/cidverseutils/filesystem v0.0.0-20240507203508-adb806ff2095/go.mod h1:rw4oLD+q/7HCMkQUooxVz4He06ORyoAn2cXLFqpBinQ= +github.com/cidverse/cidverseutils/hash v0.0.0-20240507202753-435d763dc50c h1:+WyiCMW6n+B2VnuKLIgoA7absB+VdCqQOtHG6Q2Dx2g= +github.com/cidverse/cidverseutils/hash v0.0.0-20240507202753-435d763dc50c/go.mod h1:MiRf2odL+20YA8iFNU14BT3o5d/1IOku+yjMYCxzXxo= github.com/cidverse/cidverseutils/network v0.0.0-20240501195623-588d642a5813 h1:7LjD5WKwgh/WifTMJb+iDpnCsPX75yEqBRwg8dG7jO4= github.com/cidverse/cidverseutils/network v0.0.0-20240501195623-588d642a5813/go.mod h1:LYPCE5T+fLI5LdZXwFWkVWnxheR1ppPpmxKtXKBqKsw= +github.com/cidverse/cidverseutils/redact v0.0.0-20240507202753-435d763dc50c h1:HYbiTOUvTVoMrzFqIjvGGP8V/Tfc0pFMlyq6iEh2Rnw= +github.com/cidverse/cidverseutils/redact v0.0.0-20240507202753-435d763dc50c/go.mod h1:3DuGAE15Pjh5tLSKzXwcGPCBzgTgHLMVG6WltHP+mwI= github.com/cidverse/cidverseutils/version v0.0.0-20240501195623-588d642a5813 h1:MR2ZxdeEH5gA6P9jGSmbQoaQ4BzOAHQeJ4PYukiQ4Ss= github.com/cidverse/cidverseutils/version v0.0.0-20240501195623-588d642a5813/go.mod h1:PbRN1pUYDBOT5uZXmOlQBCIoNJm6ubHJsg6Thgaq0TU= github.com/cidverse/go-rules v0.0.0-20231112122021-075e5e6f8abc h1:eVuHW4U9BoiqjIwWPATHRm6oiphVlm3Jn7QgHGgeVq8= github.com/cidverse/go-rules v0.0.0-20231112122021-075e5e6f8abc/go.mod h1:1E+4h0XkcYJVq3RHoYQradnCt7wk3JgnuRRnfeiwAzE= github.com/cidverse/go-vcs v0.0.0-20240402102656-5c7ce4c133c6 h1:MFnUQVoQj3etU3DtUK4bwN8evLcKV+KuvnRb5AlACxE= github.com/cidverse/go-vcs v0.0.0-20240402102656-5c7ce4c133c6/go.mod h1:QDjMQz4P6BH7AsBNRuLfgtSFUwmLRFozbe8h92tRhao= -github.com/cidverse/normalizeci v1.1.1-0.20240323134319-1eade1c37ec8 h1:h+0gC0H7zYh+h1S3ypgzfIt00kZL00wMoaKZDdUCIC4= -github.com/cidverse/normalizeci v1.1.1-0.20240323134319-1eade1c37ec8/go.mod h1:o4GdPz6442jndTz8RZE8NS3KCexyzkrj5O6aB0t+8iw= +github.com/cidverse/normalizeci v1.1.1-0.20240507162324-423657017129 h1:pbgchEoNEvKnIv2Hs7vmT9p+mCpr07ewjlpw0XIGzlI= +github.com/cidverse/normalizeci v1.1.1-0.20240507162324-423657017129/go.mod h1:fIJEgJ5jCHFxS61NPSpBSKXoNyMSYZC9GY3x5pVK960= github.com/cidverse/repoanalyzer v0.1.0 h1:RQZG/c/5A0B+9l8ARTO5MT2CtqNNmimdgwZPTEYMi/0= github.com/cidverse/repoanalyzer v0.1.0/go.mod h1:VmVAUm3a91TDIjDE3EwyvEnGjF9ST2RKKdBvfKoDWxE= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= @@ -98,8 +100,8 @@ github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8= -github.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk= +github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go= +github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/pkg/cmd/action.go b/pkg/cmd/action.go index afab7b9..dd10e3c 100644 --- a/pkg/cmd/action.go +++ b/pkg/cmd/action.go @@ -5,14 +5,15 @@ import ( "os" "strconv" "strings" + "sync" "text/tabwriter" "github.com/cidverse/cid/pkg/app" "github.com/cidverse/cid/pkg/common/api" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/common/workflowrun" "github.com/cidverse/cid/pkg/core/catalog" "github.com/cidverse/cid/pkg/core/rules" + "github.com/cidverse/cidverseutils/redact" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -46,7 +47,7 @@ var actionListCmd = &cobra.Command{ env := api.GetCIDEnvironment(cfg.Env, projectDir) // print list - w := tabwriter.NewWriter(protectoutput.NewProtectedWriter(nil, os.Stdout), 1, 1, 1, ' ', 0) + w := tabwriter.NewWriter(redact.NewProtectedWriter(nil, os.Stdout, &sync.Mutex{}, nil), 1, 1, 1, ' ', 0) _, _ = fmt.Fprintln(w, "REPOSITORY\tACTION\tTYPE\tSCOPE\tRULES\tDESCRIPTION") for _, action := range cfg.Registry.Actions { ruleEvaluation := "?/" + strconv.Itoa(len(action.Rules)) diff --git a/pkg/cmd/catalog.go b/pkg/cmd/catalog.go index 368c17a..eccf573 100644 --- a/pkg/cmd/catalog.go +++ b/pkg/cmd/catalog.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "os" + "sync" "text/tabwriter" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/core/catalog" + "github.com/cidverse/cidverseutils/redact" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -47,7 +48,7 @@ var catalogListCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { registries := catalog.LoadSources() // print list - w := tabwriter.NewWriter(protectoutput.NewProtectedWriter(nil, os.Stdout), 1, 1, 1, ' ', 0) + w := tabwriter.NewWriter(redact.NewProtectedWriter(nil, os.Stdout, &sync.Mutex{}, nil), 1, 1, 1, ' ', 0) _, _ = fmt.Fprintln(w, "NAME\tURI\tAdded\tUpdated\tWorkflows\tActions\tImages\tHash") for key, source := range registries { data := catalog.LoadCatalogs(map[string]*catalog.Source{key: source}) diff --git a/pkg/cmd/info.go b/pkg/cmd/info.go index ff47994..cd4ad77 100644 --- a/pkg/cmd/info.go +++ b/pkg/cmd/info.go @@ -7,7 +7,7 @@ import ( "github.com/cidverse/cid/pkg/app" "github.com/cidverse/cid/pkg/common/api" "github.com/cidverse/cid/pkg/common/command" - "github.com/cidverse/cid/pkg/common/protectoutput" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/repoanalyzer" "github.com/cidverse/repoanalyzer/analyzerapi" "github.com/rs/zerolog/log" @@ -100,6 +100,6 @@ var infoCmd = &cobra.Command{ if err != nil { log.Fatal().Err(err).Msg("failed to serialize yaml response") } - fmt.Print(protectoutput.RedactProtectedPhrases(string(responseText))) + fmt.Print(redact.Redact(string(responseText))) }, } diff --git a/pkg/cmd/module.go b/pkg/cmd/module.go index 2479186..15224c6 100644 --- a/pkg/cmd/module.go +++ b/pkg/cmd/module.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "os" + "sync" "text/tabwriter" "github.com/cidverse/cid/pkg/common/api" - "github.com/cidverse/cid/pkg/common/protectoutput" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/repoanalyzer" "github.com/rs/zerolog" "github.com/spf13/cobra" @@ -42,7 +43,7 @@ var moduleListCmd = &cobra.Command{ modules := repoanalyzer.AnalyzeProject(projectDir, projectDir) // print list - w := tabwriter.NewWriter(protectoutput.NewProtectedWriter(nil, os.Stdout), 1, 1, 1, ' ', 0) + w := tabwriter.NewWriter(redact.NewProtectedWriter(nil, os.Stdout, &sync.Mutex{}, nil), 1, 1, 1, ' ', 0) _, _ = fmt.Fprintln(w, "NAME\tBUILD-SYSTEM\tBUILD-SYNTAX\tSUBMODULES") for _, module := range modules { _, _ = fmt.Fprintln(w, module.Name+"\t"+string(module.BuildSystem)+"\t"+string(module.BuildSystemSyntax)+"\t0") diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 5d0691c..3308f55 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -3,8 +3,9 @@ package cmd import ( "os" "strings" + "sync" - "github.com/cidverse/cid/pkg/common/protectoutput" + "github.com/cidverse/cidverseutils/redact" "github.com/mattn/go-colorable" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -40,12 +41,12 @@ var rootCmd = &cobra.Command{ } var logContext zerolog.Context if cfg.LogFormat == "plain" { - logContext = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: protectoutput.NewProtectedWriter(nil, os.Stderr), NoColor: true}).With().Timestamp() + logContext = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: redact.NewProtectedWriter(nil, os.Stderr, &sync.Mutex{}, nil), NoColor: true}).With().Timestamp() } else if cfg.LogFormat == "color" { colorableOutput := colorable.NewColorableStdout() - logContext = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: protectoutput.NewProtectedWriter(nil, colorableOutput), NoColor: false}).With().Timestamp() + logContext = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: redact.NewProtectedWriter(nil, colorableOutput, &sync.Mutex{}, nil), NoColor: false}).With().Timestamp() } else if cfg.LogFormat == "json" { - logContext = zerolog.New(os.Stderr).Output(protectoutput.NewProtectedWriter(nil, os.Stderr)).With().Timestamp() + logContext = zerolog.New(os.Stderr).Output(redact.NewProtectedWriter(nil, os.Stderr, &sync.Mutex{}, nil)).With().Timestamp() } if cfg.LogCaller { logContext = logContext.Caller() diff --git a/pkg/cmd/stage.go b/pkg/cmd/stage.go index 2cadb16..2a9d9e0 100644 --- a/pkg/cmd/stage.go +++ b/pkg/cmd/stage.go @@ -4,12 +4,13 @@ import ( "fmt" "os" "strconv" + "sync" "text/tabwriter" "github.com/cidverse/cid/pkg/app" "github.com/cidverse/cid/pkg/common/api" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/core/rules" + "github.com/cidverse/cidverseutils/redact" "github.com/spf13/cobra" ) @@ -38,7 +39,7 @@ var stageListCmd = &cobra.Command{ env := api.GetCIDEnvironment(cfg.Env, projectDir) // print list - w := tabwriter.NewWriter(protectoutput.NewProtectedWriter(nil, os.Stdout), 1, 1, 1, ' ', 0) + w := tabwriter.NewWriter(redact.NewProtectedWriter(nil, os.Stdout, &sync.Mutex{}, nil), 1, 1, 1, ' ', 0) _, _ = fmt.Fprintln(w, "WORKFLOW\tSTAGE\tRULES\tACTIONS") for _, wf := range cfg.Registry.Workflows { for _, stage := range wf.Stages { diff --git a/pkg/cmd/workflow.go b/pkg/cmd/workflow.go index c5e8348..aac9397 100644 --- a/pkg/cmd/workflow.go +++ b/pkg/cmd/workflow.go @@ -4,14 +4,15 @@ import ( "fmt" "os" "strconv" + "sync" "text/tabwriter" "github.com/cidverse/cid/pkg/core/catalog" "github.com/cidverse/cid/pkg/core/provenance" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/cid/pkg/app" "github.com/cidverse/cid/pkg/common/api" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/common/workflowrun" "github.com/cidverse/cid/pkg/core/rules" "github.com/rs/zerolog/log" @@ -48,7 +49,7 @@ var workflowListCmd = &cobra.Command{ env := api.GetCIDEnvironment(cfg.Env, projectDir) // print list - w := tabwriter.NewWriter(protectoutput.NewProtectedWriter(nil, os.Stdout), 1, 1, 1, ' ', 0) + w := tabwriter.NewWriter(redact.NewProtectedWriter(nil, os.Stdout, &sync.Mutex{}, nil), 1, 1, 1, ' ', 0) _, _ = fmt.Fprintln(w, "WORKFLOW\tVERSION\tRULES\tSTAGES\tACTIONS") for _, workflow := range cfg.Registry.Workflows { _, _ = fmt.Fprintln(w, workflow.Name+"\t"+workflow.Version+"\t"+ diff --git a/pkg/common/api/api.go b/pkg/common/api/api.go index 938bd52..626468a 100644 --- a/pkg/common/api/api.go +++ b/pkg/common/api/api.go @@ -4,10 +4,10 @@ import ( "encoding/base64" "strings" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/core/config" "github.com/cidverse/cid/pkg/core/secret" "github.com/cidverse/cidverseutils/filesystem" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/normalizeci/pkg/envstruct" "github.com/cidverse/normalizeci/pkg/normalizer" "github.com/cidverse/normalizeci/pkg/normalizer/api" @@ -91,10 +91,10 @@ func AutoProtectValues(key string, original string, decoded string) { upperKey := strings.ToUpper(key) if strings.Contains(upperKey, "KEY") || strings.Contains(upperKey, "USER") || strings.Contains(upperKey, "PASS") || strings.Contains(upperKey, "PRIVATE") || strings.Contains(upperKey, "TOKEN") || strings.Contains(upperKey, "SECRET") || strings.Contains(upperKey, "AUTH") { if original != "" { - protectoutput.ProtectPhrase(original) + redact.ProtectPhrase(original) } if decoded != "" { - protectoutput.ProtectPhrase(decoded) + redact.ProtectPhrase(decoded) } } } diff --git a/pkg/common/command/command.go b/pkg/common/command/command.go index 8d70c76..bff4689 100644 --- a/pkg/common/command/command.go +++ b/pkg/common/command/command.go @@ -11,6 +11,7 @@ import ( "runtime" "sort" "strings" + "sync" "github.com/cidverse/cid/pkg/constants" "github.com/cidverse/cid/pkg/core/util" @@ -18,10 +19,10 @@ import ( "github.com/cidverse/cidverseutils/containerruntime" "github.com/cidverse/cidverseutils/filesystem" "github.com/cidverse/cidverseutils/network" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/go-vcs/vcsutil" "github.com/samber/lo" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/core/config" "github.com/rs/zerolog/log" ) @@ -59,7 +60,7 @@ func GetCommandVersion(binary string) (string, error) { // RunCommand runs a required command and forwards all output to console, but will panic/exit if the command fails func RunCommand(command string, env map[string]string, workDir string) { - err := runCommand(command, env, "", workDir, protectoutput.NewProtectedWriter(os.Stdout, nil), protectoutput.NewProtectedWriter(os.Stderr, nil)) + err := runCommand(command, env, "", workDir, redact.NewProtectedWriter(os.Stdout, nil, &sync.Mutex{}, nil), redact.NewProtectedWriter(os.Stderr, nil, &sync.Mutex{}, nil)) if err != nil { log.Fatal().Err(err).Str("command", command).Msg("failed to execute command") } @@ -67,7 +68,7 @@ func RunCommand(command string, env map[string]string, workDir string) { // RunOptionalCommand runs a command and forwards all output to console func RunOptionalCommand(command string, env map[string]string, workDir string) error { - return runCommand(command, env, "", workDir, protectoutput.NewProtectedWriter(os.Stdout, nil), protectoutput.NewProtectedWriter(os.Stderr, nil)) + return runCommand(command, env, "", workDir, redact.NewProtectedWriter(os.Stdout, nil, &sync.Mutex{}, nil), redact.NewProtectedWriter(os.Stderr, nil, &sync.Mutex{}, nil)) } // RunCommandAndGetOutput runs a command and returns the full response / command output @@ -102,11 +103,11 @@ func RunAPICommand(cmd APICommandExecute) (stdout string, stderr string, executi var stdoutBuffer bytes.Buffer var stderrBuffer bytes.Buffer if cmd.Capture { - stdoutWriter = protectoutput.NewProtectedWriter(nil, &stdoutBuffer) - stderrWriter = protectoutput.NewProtectedWriter(nil, &stderrBuffer) + stdoutWriter = redact.NewProtectedWriter(nil, &stdoutBuffer, &sync.Mutex{}, nil) + stderrWriter = redact.NewProtectedWriter(nil, &stderrBuffer, &sync.Mutex{}, nil) } else { - stdoutWriter = protectoutput.NewProtectedWriter(os.Stdout, nil) - stderrWriter = protectoutput.NewProtectedWriter(os.Stderr, nil) + stdoutWriter = redact.NewProtectedWriter(os.Stdout, nil, &sync.Mutex{}, nil) + stderrWriter = redact.NewProtectedWriter(os.Stderr, nil, &sync.Mutex{}, nil) } // identify command diff --git a/pkg/common/protectoutput/protect.go b/pkg/common/protectoutput/protect.go deleted file mode 100644 index da2fcf9..0000000 --- a/pkg/common/protectoutput/protect.go +++ /dev/null @@ -1,46 +0,0 @@ -package protectoutput - -import ( - "encoding/base64" - "strings" - - "github.com/thoas/go-funk" -) - -var protectedPhrases []string - -// ProtectPhrase will cause the provided phrase to be redacted (also base64 encoded values) -func ProtectPhrase(phrase string) { - if phrase == "" { - return - } - - if !funk.Contains(protectedPhrases, phrase) { - protectedPhrases = append(protectedPhrases, phrase) - - // add base64 decoded version, if the phrase is base64 encoded - if isBase64(phrase) { - // add base64 decoded version, if the phrase is base64 encoded - decodedValue, _ := base64.StdEncoding.DecodeString(phrase) - protectedPhrases = append(protectedPhrases, string(decodedValue)) - } - - // add base64 encoded version of the phrase - phraseBase64 := base64.StdEncoding.EncodeToString([]byte(phrase)) - protectedPhrases = append(protectedPhrases, phraseBase64) - } -} - -// RedactProtectedPhrases redacts all protected phrases in the input string (replace with ***) -func RedactProtectedPhrases(input string) string { - for _, phrase := range protectedPhrases { - input = strings.ReplaceAll(input, phrase, "[MASKED]") - } - - return input -} - -func isBase64(s string) bool { - _, err := base64.StdEncoding.DecodeString(s) - return err == nil -} diff --git a/pkg/common/protectoutput/protect_test.go b/pkg/common/protectoutput/protect_test.go deleted file mode 100644 index 346fa19..0000000 --- a/pkg/common/protectoutput/protect_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package protectoutput - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPhraseAddition(t *testing.T) { - protectedPhrases = nil - - // protect phrase - ProtectPhrase("bXlzZWNyZXQ=") - assert.Equal(t, 3, len(protectedPhrases)) - assert.Equal(t, "bXlzZWNyZXQ=", protectedPhrases[0]) - assert.Equal(t, "mysecret", protectedPhrases[1]) - assert.Equal(t, "YlhselpXTnlaWFE9", protectedPhrases[2]) -} - -func TestRedaction(t *testing.T) { - protectedPhrases = nil - - // protect phrase - ProtectPhrase("mysecret") - - // check redacted - out := RedactProtectedPhrases("abc mysecret def") - assert.Equal(t, "abc [MASKED] def", out) -} - -func TestRedactionBase64(t *testing.T) { - protectedPhrases = nil - - // protect phrase - ProtectPhrase("mysecret") - - // check redacted - out := RedactProtectedPhrases("abc bXlzZWNyZXQ= def") - assert.Equal(t, "abc [MASKED] def", out) -} - -func TestRedactionBase64Encoded(t *testing.T) { - protectedPhrases = nil - - // protect phrase - ProtectPhrase("bXlzZWNyZXQ=") - - // check redacted - out := RedactProtectedPhrases("test mysecret test") - assert.Equal(t, "test [MASKED] test", out) -} diff --git a/pkg/common/protectoutput/proxywriter.go b/pkg/common/protectoutput/proxywriter.go deleted file mode 100644 index 6d2d5b9..0000000 --- a/pkg/common/protectoutput/proxywriter.go +++ /dev/null @@ -1,53 +0,0 @@ -package protectoutput - -import ( - "io" - "os" - "sync" -) - -// lastProxyWrite contains the last write of the proxy writer for testing purposes, if no os.File or io.Writer is provided -var lastProxyWrite string - -var globalMutex = sync.Mutex{} - -type FileProxyWriter struct { - file *os.File - writer io.Writer - mutex *sync.Mutex -} - -// NewProtectedWriter proxies all output to stdout/stderr to omit/remove any kind of credentials from all logs -func NewProtectedWriter(file *os.File, writer io.Writer) *FileProxyWriter { - return &FileProxyWriter{ - file: file, - writer: writer, - mutex: &globalMutex, - } -} - -func (w *FileProxyWriter) Write(p []byte) (int, error) { - // redact protected phrases in log - output := RedactProtectedPhrases(string(p)) - - // use mutex - w.mutex.Lock() - defer w.mutex.Unlock() - - // write data - if w.file != nil { - _, err := w.file.WriteString(output) - if err != nil { - return 0, err - } - } else if w.writer != nil { - _, err := w.writer.Write([]byte(output)) - if err != nil { - return 0, err - } - } else { - lastProxyWrite = output - } - - return len(p), nil -} diff --git a/pkg/common/protectoutput/proxywriter_test.go b/pkg/common/protectoutput/proxywriter_test.go deleted file mode 100644 index 5335885..0000000 --- a/pkg/common/protectoutput/proxywriter_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package protectoutput - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNewProtectedWriter(t *testing.T) { - protectedPhrases = nil - ProtectPhrase("mySecret") - - writer := NewProtectedWriter(nil, nil) - _, _ = writer.Write([]byte("this contains a secret: mySecret")) - assert.Equal(t, "this contains a secret: [MASKED]", lastProxyWrite) -} diff --git a/pkg/core/actionexecutor/containeraction/containerexecutor.go b/pkg/core/actionexecutor/containeraction/containerexecutor.go index 0d00456..37b4e25 100644 --- a/pkg/core/actionexecutor/containeraction/containerexecutor.go +++ b/pkg/core/actionexecutor/containeraction/containerexecutor.go @@ -21,7 +21,9 @@ import ( "github.com/cidverse/cid/pkg/core/util" "github.com/cidverse/cidverseutils/ci" "github.com/cidverse/cidverseutils/containerruntime" + "github.com/cidverse/cidverseutils/hash" "github.com/cidverse/cidverseutils/network" + "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/rs/zerolog/log" ) @@ -79,7 +81,7 @@ func (e Executor) Execute(ctx *commonapi.ActionExecutionContext, localState *sta }() // create socket file - socketFile := path.Join(tempDir, util.RandomUUIDWithoutDashes()+".socket") + socketFile := path.Join(tempDir, hash.UUIDNoDash(uuid.New().String())+".socket") // listen apiEngine := restapi.Setup(restapi.APIConfig{ diff --git a/pkg/core/provenance/provenance.go b/pkg/core/provenance/provenance.go index b346ee4..66bebd8 100644 --- a/pkg/core/provenance/provenance.go +++ b/pkg/core/provenance/provenance.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/cidverse/cid/pkg/common/protectoutput" "github.com/cidverse/cid/pkg/core/state" + "github.com/cidverse/cidverseutils/redact" "github.com/cidverse/normalizeci/pkg/envstruct" nci "github.com/cidverse/normalizeci/pkg/ncispec/v1" intoto "github.com/in-toto/in-toto-golang/in_toto" @@ -64,7 +64,7 @@ func GeneratePredicate(env map[string]string, state *state.ActionStateContext) v "RUNNER": fmt.Sprintf("%s:%s", nci.Worker.Type, nci.Worker.OS), } for k, v := range env { - systemParameters[protectoutput.RedactProtectedPhrases(k)] = protectoutput.RedactProtectedPhrases(v) + systemParameters[redact.Redact(k)] = redact.Redact(v) } prov.BuildDefinition = v1.ProvenanceBuildDefinition{ BuildType: fmt.Sprintf("https://github.com/cidverse/cid@%s", "0.0.0"),