From 1d9891a2446bc5006ad949afb4ec0f1e7fc25cf6 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 4 Apr 2023 11:49:00 +0200 Subject: [PATCH 001/263] wip --- go.mod | 17 ++- go.sum | 36 ++++-- pkg/acquisition/acquisition.go | 2 + pkg/acquisition/modules/waf/waf.go | 183 ++++++++++++++++++++++++++++ pkg/cwhub/cwhub.go | 14 ++- plugins/notifications/email/go.mod | 6 +- plugins/notifications/email/go.sum | 12 +- plugins/notifications/http/go.mod | 6 +- plugins/notifications/http/go.sum | 12 +- plugins/notifications/slack/go.mod | 6 +- plugins/notifications/slack/go.sum | 12 +- plugins/notifications/splunk/go.mod | 6 +- plugins/notifications/splunk/go.sum | 12 +- 13 files changed, 263 insertions(+), 61 deletions(-) create mode 100644 pkg/acquisition/modules/waf/waf.go diff --git a/go.mod b/go.mod index 2a6e52a8ccd..dc882fa0292 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.1 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 + golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -71,6 +71,7 @@ require ( github.com/blackfireio/osinfo v1.0.3 github.com/bluele/gcache v0.0.2 github.com/cespare/xxhash/v2 v2.1.2 + github.com/corazawaf/coraza/v3 v3.0.0-rc.1 github.com/goccy/go-yaml v1.9.7 github.com/gofrs/uuid v4.0.0+incompatible github.com/golang-jwt/jwt/v4 v4.2.0 @@ -84,7 +85,7 @@ require ( github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v0.2.1 - golang.org/x/sys v0.5.0 + golang.org/x/sys v0.6.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -101,6 +102,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/containerd/containerd v1.6.18 // indirect + github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.0+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -160,6 +162,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -172,7 +175,9 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect - github.com/tidwall/gjson v1.13.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect @@ -180,10 +185,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect diff --git a/go.sum b/go.sum index 3e2fe11aee0..3c0eb6aba58 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,10 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= +github.com/corazawaf/coraza/v3 v3.0.0-rc.1 h1:Jcr5HB7eUyUaaNhmc9+Q5xZqlCzs2Bp9RHAYRzGm9ts= +github.com/corazawaf/coraza/v3 v3.0.0-rc.1/go.mod h1:GhpyYpKaOG/wHZtdyUpu74wo9StS3fzmtKvgSzms/XQ= +github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= +github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -217,6 +221,7 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= @@ -683,6 +688,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -757,6 +763,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -893,13 +901,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= -github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -1047,8 +1056,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1099,8 +1108,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1201,14 +1210,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1218,8 +1227,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1287,6 +1296,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index bf22a3fb31e..1c0d07d5fc2 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -25,6 +25,7 @@ import ( k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit" s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" + wafacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" @@ -59,6 +60,7 @@ var AcquisitionSources = map[string]func() DataSource{ "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, + "waf": func() DataSource { return &wafacquisition.WafSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go new file mode 100644 index 00000000000..b00f53d1a4c --- /dev/null +++ b/pkg/acquisition/modules/waf/waf.go @@ -0,0 +1,183 @@ +package wafacquisition + +import ( + "context" + "fmt" + "net/http" + + "github.com/corazawaf/coraza/v3" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" +) + +type WafSource struct { + config WafSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + waf coraza.WAF +} + +type WafSourceConfig struct { + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + Path string `yaml:"path"` + configuration.DataSourceCommonCfg `yaml:",inline"` +} + +/* +type DataSource interface { + GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module + GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) + UnmarshalConfig([]byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime + Configure([]byte, *log.Entry) error // Complete the YAML datasource configuration and perform runtime checks. + ConfigureByDSN(string, map[string]string, *log.Entry, string) error // Configure the datasource + GetMode() string // Get the mode (TAIL, CAT or SERVER) + GetName() string // Get the name of the module + OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) + StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file) + CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) + GetUuid() string // Get the unique identifier of the datasource + Dump() interface{} +} +*/ + +func (w *WafSource) GetMetrics() []prometheus.Collector { + return nil +} + +func (w *WafSource) GetAggregMetrics() []prometheus.Collector { + return nil +} + +func (w *WafSource) UnmarshalConfig(yamlConfig []byte) error { + wafConfig := WafSourceConfig{} + err := yaml.UnmarshalStrict(yamlConfig, &wafConfig) + if err != nil { + return errors.Wrap(err, "Cannot parse k8s-audit configuration") + } + + w.config = wafConfig + + if w.config.ListenAddr == "" { + return fmt.Errorf("listen_addr cannot be empty") + } + + if w.config.ListenPort == 0 { + return fmt.Errorf("listen_port cannot be empty") + } + + //FIXME: is that really needed ? + if w.config.Path == "" { + return fmt.Errorf("path cannot be empty") + } + + if w.config.Path[0] != '/' { + w.config.Path = "/" + w.config.Path + } + + if w.config.Mode == "" { + w.config.Mode = configuration.TAIL_MODE + } + return nil +} + +func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { + err := w.UnmarshalConfig(yamlConfig) + if err != nil { + return errors.Wrap(err, "Cannot parse waf configuration") + } + + w.logger = logger + + w.logger.Tracef("WAF configuration: %+v", w.config) + + w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) + + w.mux = http.NewServeMux() + + w.server = &http.Server{ + Addr: w.addr, + Handler: w.mux, + } + + waf, err := coraza.NewWAF(coraza.NewWAFConfig()) + + if err != nil { + return errors.Wrap(err, "Cannot create WAF") + } + + w.waf = waf + + //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec + w.mux.HandleFunc(w.config.Path, w.wafHandler) + + return nil +} + +func (w *WafSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WafSource) GetMode() string { + return w.config.Mode +} + +func (w *WafSource) GetName() string { + return "waf" +} + +func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { + w.outChan = out + t.Go(func() error { + defer types.CatchPanic("crowdsec/acquis/waf/live") + w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + t.Go(func() error { + err := w.server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return errors.Wrap(err, "WAF server failed") + } + return nil + }) + <-t.Dying() + w.logger.Infof("Stopping WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + w.server.Shutdown(context.TODO()) + return nil + }) + return nil +} + +func (w *WafSource) CanRun() error { + return nil +} + +func (w *WafSource) GetUuid() string { + return w.config.UniqueId +} + +func (w *WafSource) Dump() interface{} { + return w +} + +func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { + tx := w.waf.NewTransaction() + + if tx.IsRuleEngineOff() { + //Not enabled, just return + rw.WriteHeader(http.StatusOK) + return + } + + rw.WriteHeader(http.StatusOK) +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index f48dc223f16..a07371324cc 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -20,7 +20,8 @@ var PARSERS = "parsers" var PARSERS_OVFLW = "postoverflows" var SCENARIOS = "scenarios" var COLLECTIONS = "collections" -var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} +var WAF_RULES = "waf-rules" +var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, WAF_RULES} var hubIdx map[string]map[string]Item @@ -42,7 +43,7 @@ type ItemHubStatus struct { Status string `json:"status"` } -//Item can be : parsed, scenario, collection +// Item can be : parsed, scenario, collection type Item struct { /*descriptive info*/ Type string `yaml:"type,omitempty" json:"type,omitempty"` //parser|postoverflows|scenario|collection(|enrich) @@ -77,6 +78,7 @@ type Item struct { PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` + WafRules []string `yaml:"waf_rules,omitempty" json:"waf_rules,omitempty"` } func (i *Item) toHubStatus() ItemHubStatus { @@ -107,7 +109,7 @@ var skippedTainted = 0 var ReferenceMissingError = errors.New("Reference(s) missing in collection") var MissingHubIndex = errors.New("hub index can't be found") -//GetVersionStatus : semver requires 'v' prefix +// GetVersionStatus : semver requires 'v' prefix func GetVersionStatus(v *Item) int { return semver.Compare("v"+v.Version, "v"+v.LocalVersion) } @@ -140,7 +142,7 @@ func GetItemMap(itemType string) map[string]Item { return m } -//GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. +// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*try to resolve symlink*/ finalName := "" @@ -206,7 +208,7 @@ func DisplaySummary() { } } -//returns: human-text, Enabled, Warning, Unmanaged +// returns: human-text, Enabled, Warning, Unmanaged func ItemStatus(v Item) (string, bool, bool, bool) { strret := "disabled" Ok := false @@ -341,7 +343,7 @@ func GetInstalledCollections() ([]Item, error) { return retItems, nil } -//Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) +// Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { if _, ok := hubIdx[itemType]; !ok { log.Errorf("type %s doesn't exist", itemType) diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index da6ec43009d..142517ef376 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -20,9 +20,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index 0d87613afb5..4fb87bbb06c 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -104,8 +104,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -127,14 +127,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index 2ecd390e9c1..ff71f7145ff 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index c16fdec69a8..6ee636d6ce0 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -125,14 +125,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index cf0beedfeb7..3d6643b750a 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -22,9 +22,9 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index 09c39b4c9ca..4baf8245f7d 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -111,8 +111,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -134,14 +134,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index a8d2e34b41a..c34dde00eff 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index c16fdec69a8..6ee636d6ce0 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -125,14 +125,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 1973aa1a56699e7850596a54fd445061df5456df Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 12 Apr 2023 13:32:14 +0200 Subject: [PATCH 002/263] wip --- cmd/crowdsec-cli/main.go | 1 + cmd/crowdsec-cli/utils.go | 3 +- cmd/crowdsec-cli/waf_rules.go | 196 ++++++++++++++++++++++++++++++++++ pkg/cwhub/cwhub.go | 31 +++++- pkg/cwhub/loader.go | 4 +- 5 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 cmd/crowdsec-cli/waf_rules.go diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index eb4b1a7a1f5..17f3777b960 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -254,6 +254,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) + rootCmd.AddCommand(NewWafRulesCmd()) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 01817ac0970..cc6d7a43123 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -137,6 +137,8 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st items, err = cwhub.GetInstalledPostOverflowsAsString() case cwhub.COLLECTIONS: items, err = cwhub.GetInstalledCollectionsAsString() + case cwhub.WAF_RULES: + items, err = cwhub.GetInstalledWafRulesAsString() default: return nil, cobra.ShellCompDirectiveDefault } @@ -746,7 +748,6 @@ func getDBClient() (*database.Client, error) { return ret, nil } - func removeFromSlice(val string, slice []string) []string { var i int var value string diff --git a/cmd/crowdsec-cli/waf_rules.go b/cmd/crowdsec-cli/waf_rules.go new file mode 100644 index 00000000000..dcfbe009e2a --- /dev/null +++ b/cmd/crowdsec-cli/waf_rules.go @@ -0,0 +1,196 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewWafRulesCmd() *cobra.Command { + var cmdWafRules = &cobra.Command{ + Use: "waf-rules [action] [config]", + Short: "Install/Remove/Upgrade/Inspect waf-rule(s) from hub", + Example: `cscli waf-rules install crowdsecurity/core-rule-set +cscli waf-rules inspect crowdsecurity/core-rule-set +cscli waf-rules upgrade crowdsecurity/core-rule-set +cscli waf-rules list +cscli waf-rules remove crowdsecurity/core-rule-set +`, + Args: cobra.MinimumNArgs(1), + Aliases: []string{"waf-rule"}, + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := csConfig.LoadHub(); err != nil { + log.Fatal(err) + } + if csConfig.Hub == nil { + return fmt.Errorf("you must configure cli before interacting with hub") + } + + if err := cwhub.SetHubBranch(); err != nil { + return fmt.Errorf("error while setting hub branch: %s", err) + } + + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + log.Info("Run 'sudo cscli hub update' to get the hub index") + log.Fatalf("Failed to get Hub index : %v", err) + } + return nil + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + if cmd.Name() == "inspect" || cmd.Name() == "list" { + return + } + log.Infof(ReloadMessage()) + }, + } + + cmdWafRules.AddCommand(NewWafRulesInstallCmd()) + cmdWafRules.AddCommand(NewWafRulesRemoveCmd()) + cmdWafRules.AddCommand(NewWafRulesUpgradeCmd()) + cmdWafRules.AddCommand(NewWafRulesInspectCmd()) + cmdWafRules.AddCommand(NewWafRulesListCmd()) + + return cmdWafRules +} + +func NewWafRulesInstallCmd() *cobra.Command { + var ignoreError bool + + var cmdWafRulesInstall = &cobra.Command{ + Use: "install [config]", + Short: "Install given waf-rule(s)", + Long: `Fetch and install given waf-rule(s) from hub`, + Example: `cscli waf-rules install crowdsec/xxx crowdsec/xyz`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(cwhub.WAF_RULES, args, toComplete) + }, + Run: func(cmd *cobra.Command, args []string) { + for _, name := range args { + t := cwhub.GetItem(cwhub.WAF_RULES, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.WAF_RULES, name) + Suggest(cwhub.WAF_RULES, name, nearestItem.Name, score, ignoreError) + continue + } + if err := cwhub.InstallItem(csConfig, name, cwhub.WAF_RULES, forceAction, downloadOnly); err != nil { + if ignoreError { + log.Errorf("Error while installing '%s': %s", name, err) + } else { + log.Fatalf("Error while installing '%s': %s", name, err) + } + } + } + }, + } + cmdWafRulesInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") + cmdWafRulesInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") + cmdWafRulesInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple waf rules") + + return cmdWafRulesInstall +} + +func NewWafRulesRemoveCmd() *cobra.Command { + var cmdWafRulesRemove = &cobra.Command{ + Use: "remove [config]", + Short: "Remove given waf-rule(s)", + Long: `Remove given waf-rule(s) from hub`, + Aliases: []string{"delete"}, + Example: `cscli waf-rules remove crowdsec/xxx crowdsec/xyz`, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + }, + Run: func(cmd *cobra.Command, args []string) { + if all { + cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, "", all, purge, forceAction) + return + } + + if len(args) == 0 { + log.Fatalf("Specify at least one waf rule to remove or '--all' flag.") + } + + for _, name := range args { + cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, name, all, purge, forceAction) + } + }, + } + cmdWafRulesRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") + cmdWafRulesRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") + cmdWafRulesRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the waf rules") + + return cmdWafRulesRemove +} + +func NewWafRulesUpgradeCmd() *cobra.Command { + var cmdWafRulesUpgrade = &cobra.Command{ + Use: "upgrade [config]", + Short: "Upgrade given waf-rule(s)", + Long: `Fetch and upgrade given waf-rule(s) from hub`, + Example: `cscli waf-rules upgrade crowdsec/xxx crowdsec/xyz`, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + }, + Run: func(cmd *cobra.Command, args []string) { + if all { + cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, "", forceAction) + } else { + if len(args) == 0 { + log.Fatalf("no target waf rule to upgrade") + } + for _, name := range args { + cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, name, forceAction) + } + } + }, + } + cmdWafRulesUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the waf rules") + cmdWafRulesUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + return cmdWafRulesUpgrade +} + +func NewWafRulesInspectCmd() *cobra.Command { + var cmdWafRulesInspect = &cobra.Command{ + Use: "inspect [name]", + Short: "Inspect given waf rule", + Long: `Inspect given waf rule`, + Example: `cscli waf-rules inspect crowdsec/xxx`, + DisableAutoGenTag: true, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + }, + Run: func(cmd *cobra.Command, args []string) { + InspectItem(args[0], cwhub.WAF_RULES) + }, + } + cmdWafRulesInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") + + return cmdWafRulesInspect +} + +func NewWafRulesListCmd() *cobra.Command { + var cmdWafRulesList = &cobra.Command{ + Use: "list [name]", + Short: "List all waf rules or given one", + Long: `List all waf rules or given one`, + Example: `cscli waf-rules list +cscli waf-rules list crowdsecurity/xxx`, + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + ListItems(color.Output, []string{cwhub.WAF_RULES}, args, false, true, all) + }, + } + cmdWafRulesList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + + return cmdWafRulesList +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index a07371324cc..0e508d9f1d5 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -201,8 +201,8 @@ func AddItem(itemType string, item Item) error { } func DisplaySummary() { - log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) + log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d waf rules", len(hubIdx[COLLECTIONS]), + len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[WAF_RULES])) if skippedLocal > 0 || skippedTainted > 0 { log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) } @@ -343,6 +343,33 @@ func GetInstalledCollections() ([]Item, error) { return retItems, nil } +func GetInstalledWafRules() ([]Item, error) { + var retItems []Item + + if _, ok := hubIdx[WAF_RULES]; !ok { + return nil, fmt.Errorf("no waf rules in hubIdx") + } + for _, item := range hubIdx[WAF_RULES] { + if item.Installed { + retItems = append(retItems, item) + } + } + return retItems, nil +} + +func GetInstalledWafRulesAsString() ([]string, error) { + var retStr []string + + items, err := GetInstalledWafRules() + if err != nil { + return nil, errors.Wrap(err, "while fetching waf rules") + } + for _, it := range items { + retStr = append(retStr, it.Name) + } + return retStr, nil +} + // Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy) func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { if _, ok := hubIdx[itemType]; !ok { diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index aa63510a8e0..52b90ecae1e 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -102,7 +102,7 @@ func parser_visit(path string, f os.DirEntry, err error) error { when the collection is installed, both files are created */ //non symlinks are local user files or hub files - if f.Type() & os.ModeSymlink == 0 { + if f.Type()&os.ModeSymlink == 0 { local = true log.Tracef("%s isn't a symlink", path) } else { @@ -406,7 +406,7 @@ func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { /*if it's a collection, check its sub-items are present*/ //XX should be done later if itemType == COLLECTIONS { - var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} + var tmp = [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections, item.WafRules} for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { From d335e74c817a73926fadf969065a26d02ee9f875 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 3 May 2023 16:35:28 +0200 Subject: [PATCH 003/263] wip --- pkg/acquisition/modules/waf/waf.go | 72 ++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index b00f53d1a4c..b1255247a74 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -3,6 +3,8 @@ package wafacquisition import ( "context" "fmt" + "io" + "io/ioutil" "net/http" "github.com/corazawaf/coraza/v3" @@ -179,5 +181,75 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { return } + defer func() { + tx.ProcessLogging() + tx.Close() + }() + + tx.ProcessConnection(r.RemoteAddr, 0, "", 0) + + tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers + + for k, vr := range r.Header { + for _, v := range vr { + tx.AddRequestHeader(k, v) + } + } + + if r.Host != "" { + tx.AddRequestHeader("Host", r.Host) + // This connector relies on the host header (now host field) to populate ServerName + tx.SetServerName(r.Host) + } + + if r.TransferEncoding != nil { + tx.AddRequestHeader("Transfer-Encoding", r.TransferEncoding[0]) + } + + in := tx.ProcessRequestHeaders() + if in != nil { + w.logger.Warnf("WAF blocked request: %+v", in) + rw.WriteHeader(http.StatusForbidden) + return + } + + in = tx.ProcessRequestHeaders() + + if in != nil { + w.logger.Warnf("WAF blocked request: %+v", in) + rw.WriteHeader(http.StatusForbidden) + return + } + + if tx.IsRequestBodyAccessible() { + if r.Body != nil && r.Body != http.NoBody { + _, _, err := tx.ReadRequestBodyFrom(r.Body) + if err != nil { + w.logger.Errorf("Cannot read request body: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + bodyReader, err := tx.RequestBodyReader() + if err != nil { + w.logger.Errorf("Cannot read request body: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + body := io.MultiReader(bodyReader, r.Body) + r.Body = ioutil.NopCloser(body) + in, err = tx.ProcessRequestBody() + if err != nil { + w.logger.Errorf("Cannot process request body: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + if in != nil { + w.logger.Warnf("WAF blocked request: %+v", in) + rw.WriteHeader(http.StatusForbidden) + return + } + } + } + rw.WriteHeader(http.StatusOK) } From 1e94b24a74744b491cbf570c8bf25b1288776b38 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 4 May 2023 10:25:54 +0200 Subject: [PATCH 004/263] up --- go.mod | 16 ++++++++----- go.sum | 35 ++++++++++++++++++----------- pkg/acquisition/acquisition.go | 2 ++ plugins/notifications/email/go.mod | 4 ++-- plugins/notifications/email/go.sum | 8 +++---- plugins/notifications/http/go.mod | 4 ++-- plugins/notifications/http/go.sum | 8 +++---- plugins/notifications/slack/go.mod | 4 ++-- plugins/notifications/slack/go.sum | 8 +++---- plugins/notifications/splunk/go.mod | 4 ++-- plugins/notifications/splunk/go.sum | 8 +++---- 11 files changed, 59 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index e9d027106e0..a9689aa2989 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 + golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -70,6 +70,7 @@ require ( github.com/blackfireio/osinfo v1.0.3 github.com/bluele/gcache v0.0.2 github.com/cespare/xxhash/v2 v2.1.2 + github.com/corazawaf/coraza/v3 v3.0.0-rc.2 github.com/coreos/go-systemd/v22 v22.5.0 github.com/goccy/go-yaml v1.9.7 github.com/gofrs/uuid v4.0.0+incompatible @@ -100,6 +101,7 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.0+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -158,6 +160,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -170,7 +173,9 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect - github.com/tidwall/gjson v1.13.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect @@ -178,10 +183,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/term v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -191,6 +196,7 @@ require ( k8s.io/apimachinery v0.25.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + rsc.io/binaryregexp v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index ab0727a4cc5..080319a7659 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,10 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/corazawaf/coraza/v3 v3.0.0-rc.2 h1:nV80E4+d5qQhH8NY6SyYP7YMQpfbZ2TnZHQT29/zU6M= +github.com/corazawaf/coraza/v3 v3.0.0-rc.2/go.mod h1:TKREBLh55w3SiBbLsQpH9EFzjBAmEUH4KRaZ/kFYz20= +github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= +github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -214,6 +218,7 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -668,8 +673,6 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -680,6 +683,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -754,6 +758,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -891,13 +897,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= -github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -1045,8 +1052,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1097,8 +1104,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1206,8 +1213,8 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1217,8 +1224,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1286,6 +1293,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1456,6 +1464,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index bf22a3fb31e..1c0d07d5fc2 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -25,6 +25,7 @@ import ( k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit" s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" + wafacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" @@ -59,6 +60,7 @@ var AcquisitionSources = map[string]func() DataSource{ "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, + "waf": func() DataSource { return &wafacquisition.WafSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index 054c6018dc7..549249467ed 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -20,9 +20,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index 1c311fec37a..ace03e73be5 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -104,8 +104,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -133,8 +133,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index 97973ff4dd8..a09f74276ce 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index c5a2dcce90b..e160060b10e 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,8 +131,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index e99d3e38c2c..92640aad198 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -22,9 +22,9 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index 277d73a13f2..d5f30d5e700 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -111,8 +111,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,8 +140,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index 765d72434d7..04e4c3ccca1 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index c5a2dcce90b..e160060b10e 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,8 +131,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 53c73a5e055a4cd0827cd04bf61a693ba4e998ac Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 4 May 2023 10:26:04 +0200 Subject: [PATCH 005/263] up --- pkg/acquisition/modules/waf/waf.go | 284 +++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 pkg/acquisition/modules/waf/waf.go diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go new file mode 100644 index 00000000000..e6a0f2f1f84 --- /dev/null +++ b/pkg/acquisition/modules/waf/waf.go @@ -0,0 +1,284 @@ +package wafacquisition + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/corazawaf/coraza/v3" + corazatypes "github.com/corazawaf/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" +) + +type WafSource struct { + config WafSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + inBandWaf coraza.WAF + outOfBandWaf coraza.WAF +} + +type WafSourceConfig struct { + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + Path string `yaml:"path"` + configuration.DataSourceCommonCfg `yaml:",inline"` +} + +/* +type DataSource interface { + GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module + GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) + UnmarshalConfig([]byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime + Configure([]byte, *log.Entry) error // Complete the YAML datasource configuration and perform runtime checks. + ConfigureByDSN(string, map[string]string, *log.Entry, string) error // Configure the datasource + GetMode() string // Get the mode (TAIL, CAT or SERVER) + GetName() string // Get the name of the module + OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) + StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file) + CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) + GetUuid() string // Get the unique identifier of the datasource + Dump() interface{} +} +*/ + +func (w *WafSource) GetMetrics() []prometheus.Collector { + return nil +} + +func (w *WafSource) GetAggregMetrics() []prometheus.Collector { + return nil +} + +func (w *WafSource) UnmarshalConfig(yamlConfig []byte) error { + wafConfig := WafSourceConfig{} + err := yaml.UnmarshalStrict(yamlConfig, &wafConfig) + if err != nil { + return errors.Wrap(err, "Cannot parse waf configuration") + } + + w.config = wafConfig + + if w.config.ListenAddr == "" { + return fmt.Errorf("listen_addr cannot be empty") + } + + if w.config.ListenPort == 0 { + return fmt.Errorf("listen_port cannot be empty") + } + + //FIXME: is that really needed ? + if w.config.Path == "" { + return fmt.Errorf("path cannot be empty") + } + + if w.config.Path[0] != '/' { + w.config.Path = "/" + w.config.Path + } + + if w.config.Mode == "" { + w.config.Mode = configuration.TAIL_MODE + } + return nil +} + +func logError(error corazatypes.MatchedRule) { + msg := error.ErrorLog(0) + log.Infof("[logError][%s] %s", error.Rule().Severity(), msg) +} + +func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { + err := w.UnmarshalConfig(yamlConfig) + if err != nil { + return errors.Wrap(err, "Cannot parse waf configuration") + } + + w.logger = logger + + w.logger.Tracef("WAF configuration: %+v", w.config) + + w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) + + w.mux = http.NewServeMux() + + w.server = &http.Server{ + Addr: w.addr, + Handler: w.mux, + } + + //in-band waf : kill on sight + inbandwaf, err := coraza.NewWAF( + coraza.NewWAFConfig(). + WithErrorCallback(logError). + WithDirectivesFromFile("coraza_inband.conf"), + ) + + if err != nil { + return errors.Wrap(err, "Cannot create WAF") + } + w.inBandWaf = inbandwaf + + //out-of-band waf : log only + outofbandwaf, err := coraza.NewWAF( + coraza.NewWAFConfig(). + WithErrorCallback(logError). + WithDirectivesFromFile("coraza_outofband.conf"), + ) + if err != nil { + return errors.Wrap(err, "Cannot create WAF") + } + w.outOfBandWaf = outofbandwaf + + //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec + w.mux.HandleFunc(w.config.Path, w.wafHandler) + + return nil +} + +func (w *WafSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WafSource) GetMode() string { + return w.config.Mode +} + +func (w *WafSource) GetName() string { + return "waf" +} + +func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { + w.outChan = out + t.Go(func() error { + defer types.CatchPanic("crowdsec/acquis/waf/live") + w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + t.Go(func() error { + err := w.server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return errors.Wrap(err, "WAF server failed") + } + return nil + }) + <-t.Dying() + w.logger.Infof("Stopping WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + w.server.Shutdown(context.TODO()) + return nil + }) + return nil +} + +func (w *WafSource) CanRun() error { + return nil +} + +func (w *WafSource) GetUuid() string { + return w.config.UniqueId +} + +func (w *WafSource) Dump() interface{} { + return w +} + +func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interruption, error) { + tx := waf.NewTransaction() + + if tx.IsRuleEngineOff() { + return nil, nil + } + + defer func() { + tx.ProcessLogging() + tx.Close() + }() + + tx.ProcessConnection(r.RemoteAddr, 0, "", 0) + + tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers + + for k, vr := range r.Header { + for _, v := range vr { + tx.AddRequestHeader(k, v) + } + } + + if r.Host != "" { + tx.AddRequestHeader("Host", r.Host) + // This connector relies on the host header (now host field) to populate ServerName + tx.SetServerName(r.Host) + } + + if r.TransferEncoding != nil { + tx.AddRequestHeader("Transfer-Encoding", r.TransferEncoding[0]) + } + + in := tx.ProcessRequestHeaders() + if in != nil { + return in, nil + } + + if tx.IsRequestBodyAccessible() { + if r.Body != nil && r.Body != http.NoBody { + _, _, err := tx.ReadRequestBodyFrom(r.Body) + if err != nil { + return nil, errors.Wrap(err, "Cannot read request body") + } + bodyReader, err := tx.RequestBodyReader() + if err != nil { + return nil, errors.Wrap(err, "Cannot read request body") + + } + body := io.MultiReader(bodyReader, r.Body) + r.Body = ioutil.NopCloser(body) + in, err = tx.ProcessRequestBody() + if err != nil { + return nil, errors.Wrap(err, "Cannot process request body") + + } + if in != nil { + return in, nil + } + } + } + return nil, nil +} + +func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { + //inband first + in, err := processReqWithEngine(w.inBandWaf, r) + if err != nil { //things went south + log.Errorf("Error while processing request : %s", err) + rw.WriteHeader(http.StatusForbidden) + return + } + if in != nil { + log.Infof("Request blocked by WAF : %+v", in) + rw.WriteHeader(http.StatusForbidden) + return + } + rw.WriteHeader(http.StatusOK) + //Now we can do out of band + in2, err := processReqWithEngine(w.outOfBandWaf, r) + if err != nil { //things went south + log.Errorf("Error while processing request : %s", err) + return + } + if in2 != nil { + log.Infof("WAF triggered : %+v", in2) + return + } +} From cacdcd75b6e57c89016d42661ee1915b0cba58ac Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 4 May 2023 11:05:41 +0200 Subject: [PATCH 006/263] use fork --- go.mod | 2 ++ go.sum | 4 ++-- pkg/acquisition/modules/waf/waf.go | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a9689aa2989..8a1e0a57048 100644 --- a/go.mod +++ b/go.mod @@ -202,3 +202,5 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 + +replace github.com/corazawaf/coraza/v3 => github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7 diff --git a/go.sum b/go.sum index 080319a7659..19531d92044 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7 h1:PFVMacp5znNLuSeTXuSeRVEUXvqAuVDfab1NdwAb3+s= +github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7/go.mod h1:TKREBLh55w3SiBbLsQpH9EFzjBAmEUH4KRaZ/kFYz20= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -151,8 +153,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/corazawaf/coraza/v3 v3.0.0-rc.2 h1:nV80E4+d5qQhH8NY6SyYP7YMQpfbZ2TnZHQT29/zU6M= -github.com/corazawaf/coraza/v3 v3.0.0-rc.2/go.mod h1:TKREBLh55w3SiBbLsQpH9EFzjBAmEUH4KRaZ/kFYz20= github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index e6a0f2f1f84..f3fe77dfcee 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -206,6 +206,10 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru tx.Close() }() + //this method is not exported by coraza, so we have to do it ourselves. + //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object? + tx.RemoveRuleByID(1) + tx.ProcessConnection(r.RemoteAddr, 0, "", 0) tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers From 6ac0a9ef9da2eb7ba3155ffb8279d8752741cd62 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 5 May 2023 13:49:58 +0200 Subject: [PATCH 007/263] wip --- cmd/crowdsec-cli/utils.go | 2 ++ pkg/cwhub/loader.go | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index cc6d7a43123..81e9db7d4f3 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -332,6 +332,8 @@ func ShowMetrics(hubItem *cwhub.Item) { } ShowMetrics(hubItem) } + case cwhub.WAF_RULES: + log.Fatalf("FIXME: not implemented yet") default: log.Errorf("item of type '%s' is unknown", hubItem.Type) } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 52b90ecae1e..961e9a02e95 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -90,8 +90,11 @@ func parser_visit(path string, f os.DirEntry, err error) error { } else if stage == COLLECTIONS { ftype = COLLECTIONS stage = "" + } else if stage == WAF_RULES { + ftype = WAF_RULES + stage = "" } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { - return fmt.Errorf("unknown configuration type for file '%s'", path) + return fmt.Errorf("unknown configuration type %s for file '%s'", ftype, path) } log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype) From 6d3b2b354bcb7fcb0a435e8c208cacd1de5d14c5 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 29 May 2023 14:03:10 +0200 Subject: [PATCH 008/263] up --- go.mod | 3 ++- pkg/acquisition/modules/waf/waf.go | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a1e0a57048..af91a29c596 100644 --- a/go.mod +++ b/go.mod @@ -203,4 +203,5 @@ require ( replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 -replace github.com/corazawaf/coraza/v3 => github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7 +replace github.com/corazawaf/coraza/v3 => ../buixor-coraza + diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index f3fe77dfcee..f743767af88 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/corazawaf/coraza/v3" + "github.com/corazawaf/coraza/v3/experimental" corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -198,6 +199,7 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru tx := waf.NewTransaction() if tx.IsRuleEngineOff() { + log.Printf("engine is off") return nil, nil } @@ -207,8 +209,12 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru }() //this method is not exported by coraza, so we have to do it ourselves. - //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object? - tx.RemoveRuleByID(1) + //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ + var txx experimental.FullTransaction + + //txx := experimental.ToFullInterface(tx) + txx = tx.(experimental.FullTransaction) + txx.RemoveRuleByID(1) tx.ProcessConnection(r.RemoteAddr, 0, "", 0) @@ -232,6 +238,7 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru in := tx.ProcessRequestHeaders() if in != nil { + log.Printf("headerss") return in, nil } @@ -254,14 +261,18 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru } if in != nil { + log.Printf("nothing here") return in, nil } } } + log.Printf("done") + return nil, nil } func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { + log.Printf("yolo here %v", r) //inband first in, err := processReqWithEngine(w.inBandWaf, r) if err != nil { //things went south From abaa6a5c56a842edcb1f440d0aeec522e668aecf Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 1 Jun 2023 11:10:07 +0200 Subject: [PATCH 009/263] up --- .gitmodules | 4 ++++ coraza | 1 + go.mod | 2 +- go.sum | 2 -- test/lib/bats-assert | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) create mode 160000 coraza diff --git a/.gitmodules b/.gitmodules index 6a170946b4e..f3e8a2f5f88 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,7 @@ [submodule "tests/lib/bats-mock"] path = test/lib/bats-mock url = https://github.com/crowdsecurity/bats-mock.git +[submodule "coraza"] + path = coraza + url = http://github.com/buixor/coraza + branch = testing diff --git a/coraza b/coraza new file mode 160000 index 00000000000..063bbc322dc --- /dev/null +++ b/coraza @@ -0,0 +1 @@ +Subproject commit 063bbc322dc2ad389c354e7703638104412e197a diff --git a/go.mod b/go.mod index af91a29c596..fca6d2f6f69 100644 --- a/go.mod +++ b/go.mod @@ -203,5 +203,5 @@ require ( replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 -replace github.com/corazawaf/coraza/v3 => ../buixor-coraza +replace github.com/corazawaf/coraza/v3 => ./coraza diff --git a/go.sum b/go.sum index 19531d92044..327106a3015 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,6 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7 h1:PFVMacp5znNLuSeTXuSeRVEUXvqAuVDfab1NdwAb3+s= -github.com/buixor/coraza/v3 v3.0.1-0.20230504085753-fc713bf319a7/go.mod h1:TKREBLh55w3SiBbLsQpH9EFzjBAmEUH4KRaZ/kFYz20= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/test/lib/bats-assert b/test/lib/bats-assert index 397c735212b..44913ffe602 160000 --- a/test/lib/bats-assert +++ b/test/lib/bats-assert @@ -1 +1 @@ -Subproject commit 397c735212bf1a06cfdd0cb7806c5a6ea79582bf +Subproject commit 44913ffe6020d1561c4c4d1e26cda8e07a1f374f From 44a5c8119932d56e711650806f156fbb53499b16 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 1 Jun 2023 11:53:12 +0200 Subject: [PATCH 010/263] readme --- pkg/acquisition/modules/waf/README.md | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 pkg/acquisition/modules/waf/README.md diff --git a/pkg/acquisition/modules/waf/README.md b/pkg/acquisition/modules/waf/README.md new file mode 100644 index 00000000000..61e02f8c9ad --- /dev/null +++ b/pkg/acquisition/modules/waf/README.md @@ -0,0 +1,32 @@ +Ongoing poc for Coraza + +For config: + +coraza_inband.conf: +``` +SecRuleEngine On +SecRule ARGS:id "@eq 0" "id:1, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" +SecRequestBodyAccess On +SecRule REQUEST_BODY "@contains password" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +``` + + +coraza_outofband.conf: +``` +SecRuleEngine On +SecRule ARGS:id "@eq 2" "id:2, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" +SecRequestBodyAccess On +SecRule REQUEST_BODY "@contains totolol" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +``` + + +acquis.yaml : + +``` +listen_addr: 127.0.0.1 +listen_port: 4241 +path: / +source: waf +labels: + type: waf +``` From 65884fb4be7a714c11ca771ac948b7fccd4535bd Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 5 Jun 2023 14:22:35 +0200 Subject: [PATCH 011/263] wip --- pkg/acquisition/modules/waf/waf.go | 19 +++- pkg/waf/waf.go | 136 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 pkg/waf/waf.go diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index b1255247a74..eb21d51bbdc 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -10,6 +10,7 @@ import ( "github.com/corazawaf/coraza/v3" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -110,7 +111,23 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { Handler: w.mux, } - waf, err := coraza.NewWAF(coraza.NewWAFConfig()) + crowdsecWafConfig := waf.NewWafConfig() + + err = crowdsecWafConfig.LoadWafRules() + + if err != nil { + return fmt.Errorf("Cannot load WAF rules: %w", err) + } + + var rules string + + for _, rule := range crowdsecWafConfig.InbandRules { + rules += rule.String() + "\n" + } + + w.logger.Infof("Loading rules %+v", rules) + + waf, err := coraza.NewWAF(coraza.NewWAFConfig().WithDirectives(rules)) if err != nil { return errors.Wrap(err, "Cannot create WAF") diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go new file mode 100644 index 00000000000..f86a74199ea --- /dev/null +++ b/pkg/waf/waf.go @@ -0,0 +1,136 @@ +package waf + +import ( + "os" + "strings" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +type Hook struct { + Filter string `yaml:"filter"` + FilterExpr *vm.Program `yaml:"-"` + OnSuccess string `yaml:"on_success"` + Apply []string `yaml:"apply"` + ApplyExpr []*vm.Program +} + +type WafRule struct { + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + OnMatch []Hook `yaml:"on_match"` + MergedRules []string `yaml:"-"` + OutOfBand bool `yaml:"-"` +} + +type WafConfig struct { + InbandRules []WafRule + OutOfBandRules []WafRule +} + +func buildHook(hook Hook) (Hook, error) { + if hook.Filter != "" { + program, err := expr.Compile(hook.Filter) //FIXME: opts + if err != nil { + log.Errorf("unable to compile filter %s : %s", hook.Filter, err) + return Hook{}, err + } + hook.FilterExpr = program + } + for _, apply := range hook.Apply { + program, err := expr.Compile(apply) //FIXME: opts + if err != nil { + log.Errorf("unable to compile apply %s : %s", apply, err) + return Hook{}, err + } + hook.ApplyExpr = append(hook.ApplyExpr, program) + } + return hook, nil +} + +func (w *WafConfig) LoadWafRules() error { + var files []string + for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { + if hubWafRuleItem.Installed { + files = append(files, hubWafRuleItem.LocalPath) + } + } + log.Infof("Loading %d waf files", len(files)) + for _, file := range files { + + fileContent, err := os.ReadFile(file) //FIXME: actually read from datadir + if err != nil { + log.Errorf("unable to read file %s : %s", file, err) + continue + } + wafRule := WafRule{} + err = yaml.Unmarshal(fileContent, &wafRule) + if err != nil { + log.Errorf("unable to unmarshal file %s : %s", file, err) + continue + } + if wafRule.SecLangFilesRules != nil { + for _, rulesFile := range wafRule.SecLangFilesRules { + c, err := os.ReadFile(rulesFile) + if err != nil { + log.Errorf("unable to read file %s : %s", rulesFile, err) + continue + } + wafRule.MergedRules = append(wafRule.MergedRules, string(c)) + } + } + if wafRule.SecLangRules != nil { + wafRule.MergedRules = append(wafRule.MergedRules, wafRule.SecLangRules...) + } + + //compile hooks + for _, hook := range wafRule.OnLoad { + hook, err = buildHook(hook) + if err != nil { + log.Errorf("unable to build hook %s : %s", hook.Filter, err) + continue + } + } + + for _, hook := range wafRule.PreEval { + hook, err = buildHook(hook) + if err != nil { + log.Errorf("unable to build hook %s : %s", hook.Filter, err) + continue + } + } + + for _, hook := range wafRule.OnMatch { + hook, err = buildHook(hook) + if err != nil { + log.Errorf("unable to build hook %s : %s", hook.Filter, err) + continue + } + } + + if wafRule.MergedRules != nil { + if wafRule.OutOfBand { + w.OutOfBandRules = append(w.OutOfBandRules, wafRule) + } else { + w.InbandRules = append(w.InbandRules, wafRule) + } + } else { + log.Warnf("no rules found in file %s ??", file) + } + } + return nil +} + +func NewWafConfig() *WafConfig { + return &WafConfig{} +} + +func (w *WafRule) String() string { + return strings.Join(w.MergedRules, "\n") +} From 4a7e26af02836f9b0f076c6159791523e20e6d9f Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 5 Jun 2023 19:33:03 +0200 Subject: [PATCH 012/263] wip --- pkg/acquisition/modules/waf/waf.go | 38 ++++++++---------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 8d99d9f1d63..055e382b59c 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -8,7 +8,6 @@ import ( "net/http" "github.com/corazawaf/coraza/v3" - "github.com/corazawaf/coraza/v3/experimental" corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -39,23 +38,6 @@ type WafSourceConfig struct { configuration.DataSourceCommonCfg `yaml:",inline"` } -/* -type DataSource interface { - GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module - GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) - UnmarshalConfig([]byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime - Configure([]byte, *log.Entry) error // Complete the YAML datasource configuration and perform runtime checks. - ConfigureByDSN(string, map[string]string, *log.Entry, string) error // Configure the datasource - GetMode() string // Get the mode (TAIL, CAT or SERVER) - GetName() string // Get the name of the module - OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) - StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file) - CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) - GetUuid() string // Get the unique identifier of the datasource - Dump() interface{} -} -*/ - func (w *WafSource) GetMetrics() []prometheus.Collector { return nil } @@ -125,22 +107,22 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { err = crowdsecWafConfig.LoadWafRules() if err != nil { - return fmt.Errorf("Cannot load WAF rules: %w", err) + return fmt.Errorf("cannot load WAF rules: %w", err) } - var rules string + var inBandRules string for _, rule := range crowdsecWafConfig.InbandRules { - rules += rule.String() + "\n" + inBandRules += rule.String() + "\n" } - w.logger.Infof("Loading rules %+v", rules) + w.logger.Infof("Loading rules %+v", inBandRules) //in-band waf : kill on sight inbandwaf, err := coraza.NewWAF( coraza.NewWAFConfig(). WithErrorCallback(logError). - WithDirectives(rules), + WithDirectives(inBandRules), ) if err != nil { @@ -151,8 +133,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { //out-of-band waf : log only outofbandwaf, err := coraza.NewWAF( coraza.NewWAFConfig(). - WithErrorCallback(logError). - WithDirectivesFromFile("coraza_outofband.conf"), + WithErrorCallback(logError), //. + //WithDirectivesFromFile("coraza_outofband.conf"), ) if err != nil { return errors.Wrap(err, "Cannot create WAF") @@ -228,11 +210,11 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru //this method is not exported by coraza, so we have to do it ourselves. //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ - var txx experimental.FullTransaction + //var txx experimental.FullTransaction //txx := experimental.ToFullInterface(tx) - txx = tx.(experimental.FullTransaction) - txx.RemoveRuleByID(1) + //txx = tx.(experimental.FullTransaction) + //txx.RemoveRuleByID(1) tx.ProcessConnection(r.RemoteAddr, 0, "", 0) From d12325494986185c808136133b01926d35c0d838 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 6 Jun 2023 18:27:56 +0200 Subject: [PATCH 013/263] wip --- pkg/acquisition/modules/waf/waf.go | 116 +++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index f743767af88..013d3ee5481 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -2,16 +2,20 @@ package wafacquisition import ( "context" + "encoding/json" "fmt" "io" "io/ioutil" "net/http" + "strings" + "time" "github.com/corazawaf/coraza/v3" "github.com/corazawaf/coraza/v3/experimental" corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -195,12 +199,12 @@ func (w *WafSource) Dump() interface{} { return w } -func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interruption, error) { - tx := waf.NewTransaction() +func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*corazatypes.Interruption, corazatypes.Transaction, error) { + tx := waf.NewTransactionWithID(uuid) if tx.IsRuleEngineOff() { log.Printf("engine is off") - return nil, nil + return nil, nil, nil } defer func() { @@ -239,60 +243,146 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru in := tx.ProcessRequestHeaders() if in != nil { log.Printf("headerss") - return in, nil + return in, tx, nil } if tx.IsRequestBodyAccessible() { if r.Body != nil && r.Body != http.NoBody { _, _, err := tx.ReadRequestBodyFrom(r.Body) if err != nil { - return nil, errors.Wrap(err, "Cannot read request body") + return nil, nil, errors.Wrap(err, "Cannot read request body") } bodyReader, err := tx.RequestBodyReader() if err != nil { - return nil, errors.Wrap(err, "Cannot read request body") + return nil, nil, errors.Wrap(err, "Cannot read request body") } body := io.MultiReader(bodyReader, r.Body) r.Body = ioutil.NopCloser(body) in, err = tx.ProcessRequestBody() if err != nil { - return nil, errors.Wrap(err, "Cannot process request body") + return nil, nil, errors.Wrap(err, "Cannot process request body") } if in != nil { - log.Printf("nothing here") - return in, nil + log.Printf("exception while processing body") + return in, tx, nil } } } log.Printf("done") - return nil, nil + return nil, nil, nil +} + +func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request) ([]types.Event, error) { + evts := []types.Event{} + if tx == nil { + return nil, fmt.Errorf("tx is nil") + } + for idx, rule := range tx.MatchedRules() { + log.Printf("rule %d", idx) + evt, err := w.RuleMatchToEvent(rule, tx, r) + if err != nil { + return nil, errors.Wrap(err, "Cannot convert rule match to event") + } + evts = append(evts, evt) + } + + return evts, nil +} + +// Transforms a coraza interruption to a crowdsec event +func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r *http.Request) (types.Event, error) { + evt := types.Event{} + //we might want to change this based on in-band vs out-of-band ? + evt.Type = types.LOG + evt.ExpectMode = types.LIVE + //def needs fixing + evt.Stage = "s00-raw" + evt.Process = true + + //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. + //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers + CorazaEvent := map[string]interface{}{ + //core rule info + "rule_id": rule.Rule().ID(), + "rule_action": tx.Interruption().Action, + "rule_disruptive": rule.Disruptive(), + "rule_tags": rule.Rule().Tags(), + "rule_file": rule.Rule().File(), + "rule_file_line": rule.Rule().Line(), + "rule_revision": rule.Rule().Revision(), + "rule_secmark": rule.Rule().SecMark(), + "rule_accuracy": rule.Rule().Accuracy(), + + //http contextual infos + "upstream_addr": r.RemoteAddr, + "req_uuid": tx.ID(), + "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], + "uri": rule.URI(), + } + + corazaEventB, err := json.Marshal(CorazaEvent) + if err != nil { + return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) + } + evt.Line = types.Line{ + Time: time.Now(), + //should we add some info like listen addr/port/path ? + Labels: map[string]string{"type": "waf"}, + Process: true, + Module: "waf", + Src: "waf", + Raw: string(corazaEventB), + } + + return evt, nil } func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { log.Printf("yolo here %v", r) + //let's gen a transaction id to keep consistance accross in-band and out-of-band + uuid := uuid.New().String() //inband first - in, err := processReqWithEngine(w.inBandWaf, r) + in, tx, err := processReqWithEngine(w.inBandWaf, r, uuid) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) rw.WriteHeader(http.StatusForbidden) return } if in != nil { - log.Infof("Request blocked by WAF : %+v", in) + events, err := w.TxToEvents(tx, r) + log.Infof("Request blocked by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + log.Infof("done") + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + rw.WriteHeader(http.StatusForbidden) + return + } rw.WriteHeader(http.StatusForbidden) return } rw.WriteHeader(http.StatusOK) //Now we can do out of band - in2, err := processReqWithEngine(w.outOfBandWaf, r) + in2, tx2, err := processReqWithEngine(w.outOfBandWaf, r, uuid) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) return } if in2 != nil { + events, err := w.TxToEvents(tx2, r) + log.Infof("Request triggered by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + } + log.Infof("done") log.Infof("WAF triggered : %+v", in2) return } From 30455a8eb621cfcb12ff1cdd3d11edc619132326 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 7 Jun 2023 13:45:36 +0200 Subject: [PATCH 014/263] progress --- go.mod | 3 +-- pkg/acquisition/modules/waf/waf.go | 40 ++++++++++++++++++++---------- pkg/parser/node.go | 4 +++ pkg/parser/runtime.go | 4 ++- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 02f0defefbb..9fc6743cc1d 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 golang.org/x/crypto v0.1.0 - golang.org/x/mod v0.6.0 + golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -206,4 +206,3 @@ require ( replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 replace github.com/corazawaf/coraza/v3 => ./coraza - diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 013d3ee5481..20b1c6decc0 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -15,6 +15,8 @@ import ( corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -144,6 +146,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf + log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) + log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) @@ -170,7 +174,7 @@ func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { w.outChan = out t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/waf/live") + defer trace.CatchPanic("crowdsec/acquis/waf/live") w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) t.Go(func() error { err := w.server.ListenAndServe() @@ -200,6 +204,7 @@ func (w *WafSource) Dump() interface{} { } func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*corazatypes.Interruption, corazatypes.Transaction, error) { + var in *corazatypes.Interruption tx := waf.NewTransactionWithID(uuid) if tx.IsRuleEngineOff() { @@ -240,7 +245,8 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*coraza tx.AddRequestHeader("Transfer-Encoding", r.TransferEncoding[0]) } - in := tx.ProcessRequestHeaders() + in = tx.ProcessRequestHeaders() + //if we're inband, we should stop here, but for outofband go to the end if in != nil { log.Printf("headerss") return in, tx, nil @@ -270,19 +276,22 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*coraza } } } - log.Printf("done") - - return nil, nil, nil + log.Printf("done -> %d", len(tx.MatchedRules())) + // if in != nil { + // log.Printf("exception while processing req") + // return in, tx, nil + // } + return nil, tx, nil } -func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request) ([]types.Event, error) { +func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request, kind string) ([]types.Event, error) { evts := []types.Event{} if tx == nil { return nil, fmt.Errorf("tx is nil") } for idx, rule := range tx.MatchedRules() { log.Printf("rule %d", idx) - evt, err := w.RuleMatchToEvent(rule, tx, r) + evt, err := w.RuleMatchToEvent(rule, tx, r, kind) if err != nil { return nil, errors.Wrap(err, "Cannot convert rule match to event") } @@ -293,7 +302,7 @@ func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request) ([]t } // Transforms a coraza interruption to a crowdsec event -func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r *http.Request) (types.Event, error) { +func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r *http.Request, kind string) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -306,8 +315,9 @@ func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatype //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers CorazaEvent := map[string]interface{}{ //core rule info - "rule_id": rule.Rule().ID(), - "rule_action": tx.Interruption().Action, + "rule_type": kind, + "rule_id": rule.Rule().ID(), + //"rule_action": tx.Interruption().Action, "rule_disruptive": rule.Disruptive(), "rule_tags": rule.Rule().Tags(), "rule_file": rule.Rule().File(), @@ -323,6 +333,9 @@ func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatype "uri": rule.URI(), } + if tx.Interruption() != nil { + CorazaEvent["rule_action"] = tx.Interruption().Action + } corazaEventB, err := json.Marshal(CorazaEvent) if err != nil { return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) @@ -352,7 +365,7 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { return } if in != nil { - events, err := w.TxToEvents(tx, r) + events, err := w.TxToEvents(tx, r, "inband") log.Infof("Request blocked by WAF, %d events to send", len(events)) for _, evt := range events { w.outChan <- evt @@ -373,8 +386,9 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { log.Errorf("Error while processing request : %s", err) return } - if in2 != nil { - events, err := w.TxToEvents(tx2, r) + if tx2 != nil && len(tx2.MatchedRules()) > 0 { + log.Printf("got events and stuff to do") + events, err := w.TxToEvents(tx2, r, "outofband") log.Infof("Request triggered by WAF, %d events to send", len(events)) for _, evt := range events { w.outChan <- evt diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 84ec073bb9c..2370d37960c 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -274,6 +274,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri switch out := output.(type) { case string: gstr = out + case int: + gstr = fmt.Sprintf("%d", out) + case float64, float32: + gstr = fmt.Sprintf("%f", out) default: clog.Errorf("unexpected return type for RunTimeValue : %T", output) } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index cbeee91eeaf..3c7d382d577 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -132,6 +132,8 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er value = out case int: value = strconv.Itoa(out) + case float64, float32: + value = fmt.Sprintf("%f", out) case map[string]interface{}: clog.Warnf("Expression '%s' returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string", static.ExpValue) case []interface{}: @@ -139,7 +141,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er case nil: clog.Debugf("Expression '%s' returned nil, skipping", static.ExpValue) default: - clog.Errorf("unexpected return type for RunTimeValue : %T", output) + clog.Errorf("unexpected return type for '%s' : %T", static.ExpValue, output) return errors.New("unexpected return type for RunTimeValue") } } From 739d0863258fe82ebd378f5d740f9ac9226c080c Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 7 Jun 2023 14:12:42 +0200 Subject: [PATCH 015/263] up --- pkg/acquisition/modules/waf/README.md | 84 +++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/pkg/acquisition/modules/waf/README.md b/pkg/acquisition/modules/waf/README.md index 61e02f8c9ad..459672f5f02 100644 --- a/pkg/acquisition/modules/waf/README.md +++ b/pkg/acquisition/modules/waf/README.md @@ -3,26 +3,31 @@ Ongoing poc for Coraza For config: coraza_inband.conf: -``` +```shell SecRuleEngine On SecRule ARGS:id "@eq 0" "id:1, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" SecRequestBodyAccess On -SecRule REQUEST_BODY "@contains password" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" ``` coraza_outofband.conf: -``` +```shell SecRuleEngine On -SecRule ARGS:id "@eq 2" "id:2, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" +SecRule ARGS:id "@eq 1" "id:3,phase:1,log,msg:'Invalid id',log,auditlog" +SecRule ARGS:idd "@eq 2" "id:4,phase:1,log,msg:'Invalid id',log,auditlog" SecRequestBodyAccess On -SecRule REQUEST_BODY "@contains totolol" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +#We know that because we are not cloning the body in waf.go, the outofband rules cannot access body as it has been consumed. +#We are finding a way around this +#SecRule REQUEST_BODY "@contains totolol" "id:4, phase:2,deny,msg:'Invalid request body',log,auditlog" +#SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" + ``` acquis.yaml : -``` +```yaml listen_addr: 127.0.0.1 listen_port: 4241 path: / @@ -30,3 +35,70 @@ source: waf labels: type: waf ``` + +Coraza parser: + +```yaml +onsuccess: next_stage +debug: true +filter: "evt.Parsed.program == 'waf'" +name: crowdsecurity/waf-logs +description: "Parse WAF logs" +statics: + - parsed: cloudtrail_parsed + expression: UnmarshalJSON(evt.Line.Raw, evt.Unmarshaled, 'waf') + - meta: req_uuid + expression: evt.Unmarshaled.waf.req_uuid + - meta: source_ip + expression: evt.Unmarshaled.waf.source_ip + - meta: rule_id + expression: evt.Unmarshaled.waf.rule_id + - meta: action + expression: evt.Unmarshaled.waf.rule_action + - meta: service + value: waf + - parsed: event_type + value: waf_match + +``` + +Coraza trigger scenario: + +```yaml +type: trigger +filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "inband" +debug: true +name: coroza-triggger +description: here we go +blackhole: 2m +labels: + type: exploit + remediation: true +groupby: "evt.Meta.source_ip" +``` + +Coraza leaky scenario: + +```yaml +type: leaky +filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "outofband" +debug: true +name: coroza-leaky +description: here we go +blackhole: 2m +leakspeed: 30s +capacity: 1 +labels: + type: exploit + remediation: true +groupby: "evt.Meta.source_ip" +distinct: evt.Meta.rule_id +``` + + + +To be solved: + - We need to solve the body cloning issue + - Merge w/ hub + + From 61e1cc29d56088bbf7550c34e3814d036a28fe57 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Thu, 8 Jun 2023 17:45:21 +0200 Subject: [PATCH 016/263] update --- pkg/acquisition/modules/waf/waf.go | 201 ++++++++++++++++++++--------- 1 file changed, 138 insertions(+), 63 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 20b1c6decc0..e4ae3892666 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -4,19 +4,17 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" + "net/url" "strings" "time" "github.com/corazawaf/coraza/v3" - "github.com/corazawaf/coraza/v3/experimental" corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -25,13 +23,20 @@ import ( "gopkg.in/yaml.v2" ) +const ( + InBand = "INBAND" + OutOfBand = "OUTOFBAND" +) + type WafSource struct { - config WafSourceConfig - logger *log.Entry - mux *http.ServeMux - server *http.Server - addr string - outChan chan types.Event + config WafSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + OutOfBandChan chan ParsedRequest + inBandWaf coraza.WAF outOfBandWaf coraza.WAF } @@ -146,8 +151,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf - log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) - log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) + //log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) + //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) @@ -173,8 +178,18 @@ func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { w.outChan = out + w.OutOfBandChan = make(chan ParsedRequest) t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/waf/live") + + // start outOfBand GoRoutine + t.Go(func() error { + if err := w.ProcessOutBand(t); err != nil { + return errors.Wrap(err, "Processing Out of band routine failed: %s") + } + return nil + }) + w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) t.Go(func() error { err := w.server.ListenAndServe() @@ -203,7 +218,41 @@ func (w *WafSource) Dump() interface{} { return w } -func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*corazatypes.Interruption, corazatypes.Transaction, error) { +type ParsedRequest struct { + RemoteAddr string + Host string + Headers http.Header + URL *url.URL + Method string + Proto string + Body []byte + TransferEncoding []string + UUID string +} + +func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { + var body []byte + var err error + if r.Body != nil { + body = make([]byte, 0) + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) + } + } + return ParsedRequest{ + RemoteAddr: r.RemoteAddr, + Host: r.Host, + Headers: r.Header, + URL: r.URL, + Method: r.Method, + Proto: r.Proto, + Body: body, + TransferEncoding: r.TransferEncoding, + }, nil +} + +func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType string) (*corazatypes.Interruption, corazatypes.Transaction, error) { var in *corazatypes.Interruption tx := waf.NewTransactionWithID(uuid) @@ -219,17 +268,15 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*coraza //this method is not exported by coraza, so we have to do it ourselves. //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ - var txx experimental.FullTransaction - - //txx := experimental.ToFullInterface(tx) - txx = tx.(experimental.FullTransaction) - txx.RemoveRuleByID(1) + //var txx experimental.FullTransaction + //txx = tx.(experimental.FullTransaction) + //txx.RemoveRuleByID(1) tx.ProcessConnection(r.RemoteAddr, 0, "", 0) tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers - for k, vr := range r.Header { + for k, vr := range r.Headers { for _, v := range vr { tx.AddRequestHeader(k, v) } @@ -247,44 +294,49 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*coraza in = tx.ProcessRequestHeaders() //if we're inband, we should stop here, but for outofband go to the end - if in != nil { - log.Printf("headerss") + if in != nil && wafType == InBand { return in, tx, nil } + ct := r.Headers.Get("content-type") + if tx.IsRequestBodyAccessible() { - if r.Body != nil && r.Body != http.NoBody { - _, _, err := tx.ReadRequestBodyFrom(r.Body) + if r.Body != nil && len(r.Body) != 0 { + it, _, err := tx.WriteRequestBody(r.Body) if err != nil { return nil, nil, errors.Wrap(err, "Cannot read request body") } - bodyReader, err := tx.RequestBodyReader() - if err != nil { - return nil, nil, errors.Wrap(err, "Cannot read request body") + if it != nil { + return it, nil, nil } - body := io.MultiReader(bodyReader, r.Body) - r.Body = ioutil.NopCloser(body) + // from https://github.com/corazawaf/coraza/blob/main/internal/corazawaf/transaction.go#L419 + // urlencoded cannot end with CRLF + if ct != "application/x-www-form-urlencoded" { + it, _, err := tx.WriteRequestBody([]byte{'\r', '\n'}) + if err != nil { + return nil, nil, fmt.Errorf("cannot write to request body to buffer: %s", err.Error()) + } + + if it != nil { + return it, nil, nil + } + } + in, err = tx.ProcessRequestBody() if err != nil { return nil, nil, errors.Wrap(err, "Cannot process request body") } - if in != nil { - log.Printf("exception while processing body") + if in != nil && wafType == InBand { return in, tx, nil } } } - log.Printf("done -> %d", len(tx.MatchedRules())) - // if in != nil { - // log.Printf("exception while processing req") - // return in, tx, nil - // } return nil, tx, nil } -func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request, kind string) ([]types.Event, error) { +func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r ParsedRequest, kind string) ([]types.Event, error) { evts := []types.Event{} if tx == nil { return nil, fmt.Errorf("tx is nil") @@ -302,7 +354,7 @@ func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request, kind } // Transforms a coraza interruption to a crowdsec event -func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r *http.Request, kind string) (types.Event, error) { +func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -353,51 +405,74 @@ func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatype return evt, nil } +func (w *WafSource) ProcessOutBand(t *tomb.Tomb) error { + for { + select { + case <-t.Dying(): + log.Infof("OutOfBand function is dying") + return nil + case r := <-w.OutOfBandChan: + in2, tx2, err := processReqWithEngine(w.outOfBandWaf, r, r.UUID, OutOfBand) + if err != nil { //things went south + log.Errorf("Error while processing request : %s", err) + continue + } + if tx2 != nil && len(tx2.MatchedRules()) > 0 { + events, err := w.TxToEvents(tx2, r, OutOfBand) + log.Infof("Request triggered by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + continue + } + log.Infof("WAF triggered : %+v", in2) + } + } + } +} + func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { - log.Printf("yolo here %v", r) //let's gen a transaction id to keep consistance accross in-band and out-of-band uuid := uuid.New().String() + + // parse the request only once + parsedRequest, err := NewParsedRequestFromRequest(r) + if err != nil { + log.Errorf("%s", err) + rw.WriteHeader(http.StatusForbidden) + return + } + //inband first - in, tx, err := processReqWithEngine(w.inBandWaf, r, uuid) + in, tx, err := processReqWithEngine(w.inBandWaf, parsedRequest, uuid, InBand) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) - rw.WriteHeader(http.StatusForbidden) + rw.WriteHeader(http.StatusForbidden) // do we want to return 403 is smth went wrong ? return } + if in != nil { - events, err := w.TxToEvents(tx, r, "inband") + rw.WriteHeader(http.StatusForbidden) + events, err := w.TxToEvents(tx, parsedRequest, InBand) log.Infof("Request blocked by WAF, %d events to send", len(events)) for _, evt := range events { w.outChan <- evt } - log.Infof("done") if err != nil { log.Errorf("Cannot convert transaction to events : %s", err) - rw.WriteHeader(http.StatusForbidden) return } - rw.WriteHeader(http.StatusForbidden) return } + + // we finished the inband, we can return 200 rw.WriteHeader(http.StatusOK) - //Now we can do out of band - in2, tx2, err := processReqWithEngine(w.outOfBandWaf, r, uuid) - if err != nil { //things went south - log.Errorf("Error while processing request : %s", err) - return - } - if tx2 != nil && len(tx2.MatchedRules()) > 0 { - log.Printf("got events and stuff to do") - events, err := w.TxToEvents(tx2, r, "outofband") - log.Infof("Request triggered by WAF, %d events to send", len(events)) - for _, evt := range events { - w.outChan <- evt - } - if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - } - log.Infof("done") - log.Infof("WAF triggered : %+v", in2) - return - } + + // now we can process out of band asynchronously + go func() { + w.OutOfBandChan <- parsedRequest + }() + } From c46e2ccdad01a9682a7a539bcc7d7d740a0177b7 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 9 Jun 2023 13:00:43 +0200 Subject: [PATCH 017/263] up --- pkg/acquisition/modules/waf/waf.go | 16 ++-- pkg/csconfig/crowdsec_service.go | 4 + pkg/waf/waf.go | 116 ++++++++++++++++++++++------- pkg/waf/waf_expr_lib.go | 26 +++++++ pkg/waf/waf_helpers.go | 41 ++++++++++ 5 files changed, 170 insertions(+), 33 deletions(-) create mode 100644 pkg/waf/waf_expr_lib.go create mode 100644 pkg/waf/waf_helpers.go diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index e875e7435a3..e566e9c0693 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "os" "strings" "time" @@ -16,7 +17,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -80,6 +80,7 @@ func (w *WafSource) UnmarshalConfig(yamlConfig []byte) error { if w.config.Mode == "" { w.config.Mode = configuration.TAIL_MODE } + return nil } @@ -118,16 +119,21 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { var inBandRules string for _, rule := range crowdsecWafConfig.InbandRules { + inBandRules += rule.String() + "\n" } - w.logger.Infof("Loading rules %+v", inBandRules) + w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) + + //w.logger.Infof("Loading rules %+v", inBandRules) + + fs := os.DirFS(crowdsecWafConfig.Datadir) //in-band waf : kill on sight inbandwaf, err := coraza.NewWAF( coraza.NewWAFConfig(). WithErrorCallback(logError). - WithDirectives(inBandRules), + WithDirectives(inBandRules).WithRootFS(fs), ) if err != nil { @@ -145,8 +151,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf - log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) - log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) + //log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) + //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 2642603cf04..60484c91649 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -11,6 +11,8 @@ import ( "github.com/crowdsecurity/go-cs-lib/pkg/ptr" ) +var DataDir string // FIXME: find a better way to pass this to the waf + // CrowdsecServiceCfg contains the location of parsers/scenarios/... and acquisition files type CrowdsecServiceCfg struct { Enable *bool `yaml:"enable"` @@ -106,6 +108,8 @@ func (c *Config) LoadCrowdsec() error { c.Crowdsec.HubDir = c.ConfigPaths.HubDir c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile + DataDir = c.Crowdsec.DataDir // FIXME: find a better way to give it to the waf + if c.Crowdsec.ParserRoutinesCount <= 0 { c.Crowdsec.ParserRoutinesCount = 1 } diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index f86a74199ea..a979d9d0a4e 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -2,21 +2,29 @@ package waf import ( "os" + "path/filepath" "strings" "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) type Hook struct { - Filter string `yaml:"filter"` - FilterExpr *vm.Program `yaml:"-"` - OnSuccess string `yaml:"on_success"` - Apply []string `yaml:"apply"` - ApplyExpr []*vm.Program + Filter string `yaml:"filter"` + FilterExpr *vm.Program `yaml:"-"` + OnSuccess string `yaml:"on_success"` + Apply []string `yaml:"apply"` + ApplyExpr []*vm.Program `yaml:"-"` +} + +type CompiledHook struct { + Filter *vm.Program `yaml:"-"` + Apply []*vm.Program `yaml:"-"` } type WafRule struct { @@ -25,33 +33,43 @@ type WafRule struct { OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` - MergedRules []string `yaml:"-"` - OutOfBand bool `yaml:"-"` + + CompiledOnLoad []CompiledHook `yaml:"-"` + CompiledPreEval []CompiledHook `yaml:"-"` + CompiledOnMatch []CompiledHook `yaml:"-"` + + MergedRules []string `yaml:"-"` + OutOfBand bool `yaml:"-"` } type WafConfig struct { InbandRules []WafRule OutOfBandRules []WafRule + Datadir string + logger *log.Entry } -func buildHook(hook Hook) (Hook, error) { +func buildHook(hook Hook) (CompiledHook, error) { + compiledHook := CompiledHook{} if hook.Filter != "" { program, err := expr.Compile(hook.Filter) //FIXME: opts if err != nil { log.Errorf("unable to compile filter %s : %s", hook.Filter, err) - return Hook{}, err + return CompiledHook{}, err } - hook.FilterExpr = program + compiledHook.Filter = program } for _, apply := range hook.Apply { - program, err := expr.Compile(apply) //FIXME: opts + program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ + "WafRules": []WafRule{}, + })...) if err != nil { log.Errorf("unable to compile apply %s : %s", apply, err) - return Hook{}, err + return CompiledHook{}, err } - hook.ApplyExpr = append(hook.ApplyExpr, program) + compiledHook.Apply = append(compiledHook.Apply, program) } - return hook, nil + return compiledHook, nil } func (w *WafConfig) LoadWafRules() error { @@ -61,28 +79,37 @@ func (w *WafConfig) LoadWafRules() error { files = append(files, hubWafRuleItem.LocalPath) } } - log.Infof("Loading %d waf files", len(files)) + w.logger.Infof("Loading %d waf files", len(files)) for _, file := range files { fileContent, err := os.ReadFile(file) //FIXME: actually read from datadir if err != nil { - log.Errorf("unable to read file %s : %s", file, err) + w.logger.Errorf("unable to read file %s : %s", file, err) continue } wafRule := WafRule{} err = yaml.Unmarshal(fileContent, &wafRule) if err != nil { - log.Errorf("unable to unmarshal file %s : %s", file, err) + w.logger.Errorf("unable to unmarshal file %s : %s", file, err) continue } if wafRule.SecLangFilesRules != nil { for _, rulesFile := range wafRule.SecLangFilesRules { - c, err := os.ReadFile(rulesFile) + fullPath := filepath.Join(w.Datadir, rulesFile) + c, err := os.ReadFile(fullPath) if err != nil { - log.Errorf("unable to read file %s : %s", rulesFile, err) + w.logger.Errorf("unable to read file %s : %s", rulesFile, err) continue } - wafRule.MergedRules = append(wafRule.MergedRules, string(c)) + for _, line := range strings.Split(string(c), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + wafRule.MergedRules = append(wafRule.MergedRules, line) + } } } if wafRule.SecLangRules != nil { @@ -91,27 +118,48 @@ func (w *WafConfig) LoadWafRules() error { //compile hooks for _, hook := range wafRule.OnLoad { - hook, err = buildHook(hook) + compiledHook, err := buildHook(hook) if err != nil { - log.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) continue } + wafRule.CompiledOnLoad = append(wafRule.CompiledOnLoad, compiledHook) } for _, hook := range wafRule.PreEval { - hook, err = buildHook(hook) + compiledHook, err := buildHook(hook) if err != nil { - log.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) continue } + wafRule.CompiledPreEval = append(wafRule.CompiledPreEval, compiledHook) } for _, hook := range wafRule.OnMatch { - hook, err = buildHook(hook) + compiledHook, err := buildHook(hook) if err != nil { - log.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) continue } + wafRule.CompiledOnMatch = append(wafRule.CompiledOnMatch, compiledHook) + } + + //Run the on_load hooks + + if len(wafRule.CompiledOnLoad) > 0 { + w.logger.Infof("Running %d on_load hooks", len(wafRule.CompiledOnLoad)) + for hookIdx, onLoadHook := range wafRule.CompiledOnLoad { + //Ignore filter for on load ? + if onLoadHook.Apply != nil { + for exprIdx, applyExpr := range onLoadHook.Apply { + _, err := expr.Run(applyExpr, nil) //FIXME: give proper env + if err != nil { + w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafRule.OnLoad[hookIdx].Apply[exprIdx], err) + continue + } + } + } + } } if wafRule.MergedRules != nil { @@ -121,14 +169,26 @@ func (w *WafConfig) LoadWafRules() error { w.InbandRules = append(w.InbandRules, wafRule) } } else { - log.Warnf("no rules found in file %s ??", file) + w.logger.Warnf("no rules found in file %s ??", file) } } return nil } func NewWafConfig() *WafConfig { - return &WafConfig{} + //FIXME: find a better way to get the datadir + clog := log.New() + if err := types.ConfigureLogger(clog); err != nil { + //return nil, fmt.Errorf("while configuring datasource logger: %w", err) + return nil + } + logger := clog.WithFields(log.Fields{ + "type": "waf-config", + }) + + initWafHelpers() + + return &WafConfig{Datadir: csconfig.DataDir, logger: logger} } func (w *WafRule) String() string { diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go new file mode 100644 index 00000000000..79d69345861 --- /dev/null +++ b/pkg/waf/waf_expr_lib.go @@ -0,0 +1,26 @@ +package waf + +//This is a copy paste from expr_lib.go, we probably want to only have one ? + +type exprCustomFunc struct { + name string + function func(params ...any) (any, error) + signature []interface{} +} + +var exprFuncs = []exprCustomFunc{ + { + name: "SetRulesToInband", + function: SetRulesToInband, + signature: []interface{}{ + new(func() error), + }, + }, + { + name: "SetRulesToOutOfBand", + function: SetRulesToOutOfBand, + signature: []interface{}{ + new(func() error), + }, + }, +} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go new file mode 100644 index 00000000000..b99732ca3b5 --- /dev/null +++ b/pkg/waf/waf_helpers.go @@ -0,0 +1,41 @@ +package waf + +import ( + "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" +) + +var exprFunctionOptions []expr.Option + +func initWafHelpers() { + exprFunctionOptions = []expr.Option{} + for _, function := range exprFuncs { + exprFunctionOptions = append(exprFunctionOptions, + expr.Function(function.name, + function.function, + function.signature..., + )) + } +} + +func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { + baseHelpers := exprhelpers.GetExprOptions(ctx) + + for _, function := range exprFuncs { + baseHelpers = append(baseHelpers, + expr.Function(function.name, + function.function, + function.signature..., + )) + } + return baseHelpers +} + +func SetRulesToInband(params ...any) (any, error) { + + return nil, nil +} + +func SetRulesToOutOfBand(params ...any) (any, error) { + return nil, nil +} From fa172bed56968114dfbbc6948939e3fc97045fc6 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 13 Jun 2023 15:41:32 +0200 Subject: [PATCH 018/263] up --- pkg/acquisition/modules/waf/waf.go | 67 ++++++++++++++++-------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 8a8eea2f62a..06ef34228dc 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -26,8 +26,8 @@ import ( ) const ( - InBand = "INBAND" - OutOfBand = "OUTOFBAND" + InBand = "inband" + OutOfBand = "outofband" ) type WafSource struct { @@ -124,12 +124,10 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } var inBandRules string - for _, rule := range crowdsecWafConfig.InbandRules { inBandRules += rule.String() + "\n" } - w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) //w.logger.Infof("Loading rules %+v", inBandRules) @@ -148,17 +146,23 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.inBandWaf = inbandwaf + var outOfBandRules string + for _, rule := range crowdsecWafConfig.OutOfBandRules { + outOfBandRules += rule.String() + "\n" + } + + w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) //out-of-band waf : log only outofbandwaf, err := coraza.NewWAF( coraza.NewWAFConfig(). - WithErrorCallback(logError), //. - //WithDirectivesFromFile("coraza_outofband.conf"), + WithErrorCallback(logError). + WithDirectives(outOfBandRules).WithRootFS(fs), ) + if err != nil { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf - //log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec @@ -302,6 +306,7 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } in = tx.ProcessRequestHeaders() + //if we're inband, we should stop here, but for outofband go to the end if in != nil && wafType == InBand { return in, tx, nil @@ -309,39 +314,38 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType ct := r.Headers.Get("content-type") - if tx.IsRequestBodyAccessible() { - if r.Body != nil && len(r.Body) != 0 { - it, _, err := tx.WriteRequestBody(r.Body) + if r.Body != nil && len(r.Body) != 0 { + it, _, err := tx.WriteRequestBody(r.Body) + if err != nil { + return nil, nil, errors.Wrap(err, "Cannot read request body") + } + + if it != nil { + return it, nil, nil + } + // from https://github.com/corazawaf/coraza/blob/main/internal/corazawaf/transaction.go#L419 + // urlencoded cannot end with CRLF + if ct != "application/x-www-form-urlencoded" { + it, _, err := tx.WriteRequestBody([]byte{'\r', '\n'}) if err != nil { - return nil, nil, errors.Wrap(err, "Cannot read request body") + return nil, nil, fmt.Errorf("cannot write to request body to buffer: %s", err.Error()) } if it != nil { return it, nil, nil } - // from https://github.com/corazawaf/coraza/blob/main/internal/corazawaf/transaction.go#L419 - // urlencoded cannot end with CRLF - if ct != "application/x-www-form-urlencoded" { - it, _, err := tx.WriteRequestBody([]byte{'\r', '\n'}) - if err != nil { - return nil, nil, fmt.Errorf("cannot write to request body to buffer: %s", err.Error()) - } - - if it != nil { - return it, nil, nil - } - } + } + } - in, err = tx.ProcessRequestBody() - if err != nil { - return nil, nil, errors.Wrap(err, "Cannot process request body") + in, err := tx.ProcessRequestBody() + if err != nil { + return nil, nil, errors.Wrap(err, "Cannot process request body") + } - } - if in != nil && wafType == InBand { - return in, tx, nil - } - } + if in != nil && wafType == InBand { + return in, tx, nil } + return nil, tx, nil } @@ -478,7 +482,6 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { // we finished the inband, we can return 200 rw.WriteHeader(http.StatusOK) - // now we can process out of band asynchronously go func() { w.OutOfBandChan <- parsedRequest From 40f65de7b918e3ac77839f01ef06182441f0db15 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 13 Jun 2023 16:31:30 +0200 Subject: [PATCH 019/263] optim --- pkg/acquisition/modules/waf/waf.go | 61 +++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 06ef34228dc..df9577aa794 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -38,6 +38,7 @@ type WafSource struct { addr string outChan chan types.Event OutOfBandChan chan ParsedRequest + InBandChan chan ParsedRequest inBandWaf coraza.WAF outOfBandWaf coraza.WAF @@ -190,6 +191,7 @@ func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { w.outChan = out w.OutOfBandChan = make(chan ParsedRequest) + w.InBandChan = make(chan ParsedRequest) t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/waf/live") @@ -201,6 +203,14 @@ func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) err return nil }) + // start InBand GoRoutine + t.Go(func() error { + if err := w.ProcessInBand(t); err != nil { + return errors.Wrap(err, "Processing in-band routine failed: %s") + } + return nil + }) + w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) t.Go(func() error { err := w.server.ListenAndServe() @@ -239,6 +249,7 @@ type ParsedRequest struct { Body []byte TransferEncoding []string UUID string + Tx corazatypes.Transaction } func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { @@ -349,14 +360,14 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType return nil, tx, nil } -func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r ParsedRequest, kind string) ([]types.Event, error) { +func (w *WafSource) TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { evts := []types.Event{} - if tx == nil { + if r.Tx == nil { return nil, fmt.Errorf("tx is nil") } - for idx, rule := range tx.MatchedRules() { + for idx, rule := range r.Tx.MatchedRules() { log.Printf("rule %d", idx) - evt, err := w.RuleMatchToEvent(rule, tx, r, kind) + evt, err := w.RuleMatchToEvent(rule, r.Tx, r, kind) if err != nil { return nil, errors.Wrap(err, "Cannot convert rule match to event") } @@ -430,8 +441,9 @@ func (w *WafSource) ProcessOutBand(t *tomb.Tomb) error { log.Errorf("Error while processing request : %s", err) continue } + r.Tx = tx2 if tx2 != nil && len(tx2.MatchedRules()) > 0 { - events, err := w.TxToEvents(tx2, r, OutOfBand) + events, err := w.TxToEvents(r, OutOfBand) log.Infof("Request triggered by WAF, %d events to send", len(events)) for _, evt := range events { w.outChan <- evt @@ -446,6 +458,27 @@ func (w *WafSource) ProcessOutBand(t *tomb.Tomb) error { } } +func (w *WafSource) ProcessInBand(t *tomb.Tomb) error { + for { + select { + case <-t.Dying(): + log.Infof("OutOfBand function is dying") + return nil + case r := <-w.InBandChan: + events, err := w.TxToEvents(r, InBand) + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + continue + } + + log.Infof("Request blocked by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + } + } +} + func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { //let's gen a transaction id to keep consistance accross in-band and out-of-band uuid := uuid.New().String() @@ -468,23 +501,17 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { if in != nil { rw.WriteHeader(http.StatusForbidden) - events, err := w.TxToEvents(tx, parsedRequest, InBand) - log.Infof("Request blocked by WAF, %d events to send", len(events)) - for _, evt := range events { - w.outChan <- evt - } - if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - return - } - return + go func() { + parsedRequest.Tx = tx + w.InBandChan <- parsedRequest + }() } // we finished the inband, we can return 200 rw.WriteHeader(http.StatusOK) - // now we can process out of band asynchronously + + //now we can process out of band asynchronously go func() { w.OutOfBandChan <- parsedRequest }() - } From 805752dc62a4dae84350999faf67d7d5ebf4e191 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 13 Jun 2023 17:07:42 +0200 Subject: [PATCH 020/263] wip --- pkg/acquisition/modules/waf/waf.go | 17 +++++++++++++++++ pkg/waf/waf.go | 8 ++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index df9577aa794..883e1c3af3a 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -17,6 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -142,6 +143,14 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { WithDirectives(inBandRules).WithRootFS(fs), ) + //for _, rule := range inbandwaf.GetWAF().Rules.GetRules() { + // w.logger.Infof("Action for Rule %d: %+v ", rule.ID(), rule.GetActions()) + //} + + //betterwaf := experimental.ToBetterWAFEngine(inbandwaf) + + //spew.Dump(betterwaf.Waf.Rules) + if err != nil { return errors.Wrap(err, "Cannot create WAF") } @@ -288,6 +297,8 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType tx.Close() }() + log.Infof("Processing request with %s WAF", wafType) + //this method is not exported by coraza, so we have to do it ourselves. //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ //var txx experimental.FullTransaction @@ -317,6 +328,12 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } in = tx.ProcessRequestHeaders() + //spew.Dump(in) + //spew.Dump(tx.MatchedRules()) + + for _, rule := range tx.MatchedRules() { + spew.Dump(rule.Rule()) + } //if we're inband, we should stop here, but for outofband go to the end if in != nil && wafType == InBand { diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index a979d9d0a4e..32ed9c2689d 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -61,7 +61,8 @@ func buildHook(hook Hook) (CompiledHook, error) { } for _, apply := range hook.Apply { program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ - "WafRules": []WafRule{}, + "InBandRules": []WafRule{}, + "OutOfBandRules": []WafRule{}, })...) if err != nil { log.Errorf("unable to compile apply %s : %s", apply, err) @@ -152,7 +153,10 @@ func (w *WafConfig) LoadWafRules() error { //Ignore filter for on load ? if onLoadHook.Apply != nil { for exprIdx, applyExpr := range onLoadHook.Apply { - _, err := expr.Run(applyExpr, nil) //FIXME: give proper env + _, err := expr.Run(applyExpr, map[string]interface{}{ + "InBandRules": []WafRule{}, + "OutOfBandRules": []WafRule{}, + }) if err != nil { w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafRule.OnLoad[hookIdx].Apply[exprIdx], err) continue From 9180ac7be9060340cd6242d066a1cce0c1dc5cf4 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 15 Jun 2023 22:51:57 +0200 Subject: [PATCH 021/263] wip --- pkg/acquisition/modules/waf/waf.go | 34 +++----- pkg/waf/waf.go | 134 ++++++++++++++--------------- pkg/waf/waf_expr_lib.go | 4 +- pkg/waf/waf_rules_collection.go | 35 ++++++++ 4 files changed, 117 insertions(+), 90 deletions(-) create mode 100644 pkg/waf/waf_rules_collection.go diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 883e1c3af3a..ad72d582c26 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -117,24 +117,32 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { Handler: w.mux, } - crowdsecWafConfig := waf.NewWafConfig() + ruleLoader := waf.NewWafRuleLoader() - err = crowdsecWafConfig.LoadWafRules() + rulesCollections, err := ruleLoader.LoadWafRules() if err != nil { return fmt.Errorf("cannot load WAF rules: %w", err) } var inBandRules string - for _, rule := range crowdsecWafConfig.InbandRules { + var outOfBandRules string + + //spew.Dump(rulesCollections) - inBandRules += rule.String() + "\n" + for _, collection := range rulesCollections { + if !collection.OutOfBand { + inBandRules += collection.String() + "\n" + } else { + outOfBandRules += collection.String() + "\n" + } } + w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) //w.logger.Infof("Loading rules %+v", inBandRules) - fs := os.DirFS(crowdsecWafConfig.Datadir) + fs := os.DirFS(ruleLoader.Datadir) //in-band waf : kill on sight inbandwaf, err := coraza.NewWAF( @@ -143,24 +151,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { WithDirectives(inBandRules).WithRootFS(fs), ) - //for _, rule := range inbandwaf.GetWAF().Rules.GetRules() { - // w.logger.Infof("Action for Rule %d: %+v ", rule.ID(), rule.GetActions()) - //} - - //betterwaf := experimental.ToBetterWAFEngine(inbandwaf) - - //spew.Dump(betterwaf.Waf.Rules) - if err != nil { return errors.Wrap(err, "Cannot create WAF") } w.inBandWaf = inbandwaf - var outOfBandRules string - for _, rule := range crowdsecWafConfig.OutOfBandRules { - outOfBandRules += rule.String() + "\n" - } - w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) //out-of-band waf : log only outofbandwaf, err := coraza.NewWAF( @@ -173,9 +168,6 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf - //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) - - //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) return nil diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index 32ed9c2689d..97bfc997cb9 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -1,6 +1,7 @@ package waf import ( + "fmt" "os" "path/filepath" "strings" @@ -10,6 +11,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -27,26 +29,25 @@ type CompiledHook struct { Apply []*vm.Program `yaml:"-"` } -type WafRule struct { +/*type WafConfig struct { + InbandRules []WafRule + OutOfBandRules []WafRule + Datadir string + logger *log.Entry +}*/ + +// This represents one "waf-rule" config +type WafConfig struct { SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` - - CompiledOnLoad []CompiledHook `yaml:"-"` - CompiledPreEval []CompiledHook `yaml:"-"` - CompiledOnMatch []CompiledHook `yaml:"-"` - - MergedRules []string `yaml:"-"` - OutOfBand bool `yaml:"-"` } -type WafConfig struct { - InbandRules []WafRule - OutOfBandRules []WafRule - Datadir string - logger *log.Entry +type WafRuleLoader struct { + logger *log.Entry + Datadir string } func buildHook(hook Hook) (CompiledHook, error) { @@ -54,48 +55,56 @@ func buildHook(hook Hook) (CompiledHook, error) { if hook.Filter != "" { program, err := expr.Compile(hook.Filter) //FIXME: opts if err != nil { - log.Errorf("unable to compile filter %s : %s", hook.Filter, err) - return CompiledHook{}, err + return CompiledHook{}, fmt.Errorf("unable to compile filter %s : %w", hook.Filter, err) } compiledHook.Filter = program } for _, apply := range hook.Apply { program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ - "InBandRules": []WafRule{}, - "OutOfBandRules": []WafRule{}, + "rules": &WafRulesCollection{}, })...) if err != nil { - log.Errorf("unable to compile apply %s : %s", apply, err) - return CompiledHook{}, err + return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err) } compiledHook.Apply = append(compiledHook.Apply, program) } return compiledHook, nil } -func (w *WafConfig) LoadWafRules() error { - var files []string +func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { + var wafRulesFiles []string for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { if hubWafRuleItem.Installed { - files = append(files, hubWafRuleItem.LocalPath) + wafRulesFiles = append(wafRulesFiles, hubWafRuleItem.LocalPath) } } - w.logger.Infof("Loading %d waf files", len(files)) - for _, file := range files { - fileContent, err := os.ReadFile(file) //FIXME: actually read from datadir + if len(wafRulesFiles) == 0 { + return nil, fmt.Errorf("no waf rules found in hub") + } + + w.logger.Infof("Loading %d waf files", len(wafRulesFiles)) + wafRulesCollections := []*WafRulesCollection{} + for _, wafRulesFile := range wafRulesFiles { + + fileContent, err := os.ReadFile(wafRulesFile) if err != nil { - w.logger.Errorf("unable to read file %s : %s", file, err) + w.logger.Errorf("unable to read file %s : %s", wafRulesFile, err) continue } - wafRule := WafRule{} - err = yaml.Unmarshal(fileContent, &wafRule) + wafConfig := WafConfig{} + err = yaml.Unmarshal(fileContent, &wafConfig) if err != nil { - w.logger.Errorf("unable to unmarshal file %s : %s", file, err) + w.logger.Errorf("unable to unmarshal file %s : %s", wafRulesFile, err) continue } - if wafRule.SecLangFilesRules != nil { - for _, rulesFile := range wafRule.SecLangFilesRules { + + spew.Dump(wafConfig) + + collection := &WafRulesCollection{} + + if wafConfig.SecLangFilesRules != nil { + for _, rulesFile := range wafConfig.SecLangFilesRules { fullPath := filepath.Join(w.Datadir, rulesFile) c, err := os.ReadFile(fullPath) if err != nil { @@ -109,77 +118,72 @@ func (w *WafConfig) LoadWafRules() error { if strings.TrimSpace(line) == "" { continue } - wafRule.MergedRules = append(wafRule.MergedRules, line) + collection.Rules = append(collection.Rules, WafRule{RawRule: line}) } } } - if wafRule.SecLangRules != nil { - wafRule.MergedRules = append(wafRule.MergedRules, wafRule.SecLangRules...) + + if wafConfig.SecLangRules != nil { + for _, rule := range wafConfig.SecLangRules { + collection.Rules = append(collection.Rules, WafRule{RawRule: rule}) + } } + //TODO: add our own format + //compile hooks - for _, hook := range wafRule.OnLoad { + for _, hook := range wafConfig.OnLoad { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build on_load hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledOnLoad = append(wafRule.CompiledOnLoad, compiledHook) + collection.CompiledOnLoad = append(collection.CompiledOnLoad, compiledHook) } - for _, hook := range wafRule.PreEval { + for _, hook := range wafConfig.PreEval { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build pre_eval hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledPreEval = append(wafRule.CompiledPreEval, compiledHook) + collection.CompiledPreEval = append(collection.CompiledPreEval, compiledHook) } - for _, hook := range wafRule.OnMatch { + for _, hook := range wafConfig.OnMatch { compiledHook, err := buildHook(hook) if err != nil { - w.logger.Errorf("unable to build hook %s : %s", hook.Filter, err) + w.logger.Errorf("unable to build on_match hook %s : %s", hook.Filter, err) continue } - wafRule.CompiledOnMatch = append(wafRule.CompiledOnMatch, compiledHook) + collection.CompiledOnMatch = append(collection.CompiledOnMatch, compiledHook) } //Run the on_load hooks - - if len(wafRule.CompiledOnLoad) > 0 { - w.logger.Infof("Running %d on_load hooks", len(wafRule.CompiledOnLoad)) - for hookIdx, onLoadHook := range wafRule.CompiledOnLoad { + if len(collection.CompiledOnLoad) > 0 { + w.logger.Infof("Running %d on_load hooks", len(collection.CompiledOnLoad)) + for hookIdx, onLoadHook := range collection.CompiledOnLoad { //Ignore filter for on load ? if onLoadHook.Apply != nil { for exprIdx, applyExpr := range onLoadHook.Apply { _, err := expr.Run(applyExpr, map[string]interface{}{ - "InBandRules": []WafRule{}, - "OutOfBandRules": []WafRule{}, + "rules": collection, }) if err != nil { - w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafRule.OnLoad[hookIdx].Apply[exprIdx], err) + w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) continue } } } } } - - if wafRule.MergedRules != nil { - if wafRule.OutOfBand { - w.OutOfBandRules = append(w.OutOfBandRules, wafRule) - } else { - w.InbandRules = append(w.InbandRules, wafRule) - } - } else { - w.logger.Warnf("no rules found in file %s ??", file) - } + wafRulesCollections = append(wafRulesCollections, collection) } - return nil + + return wafRulesCollections, nil } -func NewWafConfig() *WafConfig { +func NewWafRuleLoader() *WafRuleLoader { //FIXME: find a better way to get the datadir clog := log.New() if err := types.ConfigureLogger(clog); err != nil { @@ -192,9 +196,5 @@ func NewWafConfig() *WafConfig { initWafHelpers() - return &WafConfig{Datadir: csconfig.DataDir, logger: logger} -} - -func (w *WafRule) String() string { - return strings.Join(w.MergedRules, "\n") + return &WafRuleLoader{Datadir: csconfig.DataDir, logger: logger} } diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 79d69345861..33dd1f4e4ee 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -9,7 +9,7 @@ type exprCustomFunc struct { } var exprFuncs = []exprCustomFunc{ - { + /*{ name: "SetRulesToInband", function: SetRulesToInband, signature: []interface{}{ @@ -22,5 +22,5 @@ var exprFuncs = []exprCustomFunc{ signature: []interface{}{ new(func() error), }, - }, + },*/ } diff --git a/pkg/waf/waf_rules_collection.go b/pkg/waf/waf_rules_collection.go new file mode 100644 index 00000000000..3ce130e5645 --- /dev/null +++ b/pkg/waf/waf_rules_collection.go @@ -0,0 +1,35 @@ +package waf + +import "strings" + +type WafRule struct { + RawRule string +} + +// This is the "compiled" state of a WafConfig +type WafRulesCollection struct { + Rules []WafRule + CompiledOnLoad []CompiledHook `yaml:"-"` + CompiledPreEval []CompiledHook `yaml:"-"` + CompiledOnMatch []CompiledHook `yaml:"-"` + OutOfBand bool +} + +func (w *WafRulesCollection) SetInBand() error { + w.OutOfBand = false + return nil +} + +func (w *WafRulesCollection) SetOutOfBand() error { + w.OutOfBand = true + return nil +} + +func (w *WafRulesCollection) String() string { + //return strings.Join(w.Rules, "\n") + var rules []string + for _, rule := range w.Rules { + rules = append(rules, rule.RawRule) + } + return strings.Join(rules, "\n") +} From 07b60233db426838c1a7309c580aad0cbcbe1e31 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Fri, 16 Jun 2023 12:19:08 +0200 Subject: [PATCH 022/263] update waf --- pkg/acquisition/modules/waf/utils.go | 81 +++++++ pkg/acquisition/modules/waf/waf.go | 351 ++++++++++++++------------- pkg/waf/waf.go | 10 + 3 files changed, 269 insertions(+), 173 deletions(-) create mode 100644 pkg/acquisition/modules/waf/utils.go diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go new file mode 100644 index 00000000000..5e8431b2964 --- /dev/null +++ b/pkg/acquisition/modules/waf/utils.go @@ -0,0 +1,81 @@ +package wafacquisition + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + corazatypes "github.com/corazawaf/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" +) + +func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { + evts := []types.Event{} + if r.Tx == nil { + return nil, fmt.Errorf("tx is nil") + } + for _, rule := range r.Tx.MatchedRules() { + //log.Printf("rule %d", idx) + evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) + if err != nil { + return nil, errors.Wrap(err, "Cannot convert rule match to event") + } + evts = append(evts, evt) + } + + return evts, nil +} + +// Transforms a coraza interruption to a crowdsec event +func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) { + evt := types.Event{} + //we might want to change this based on in-band vs out-of-band ? + evt.Type = types.LOG + evt.ExpectMode = types.LIVE + //def needs fixing + evt.Stage = "s00-raw" + evt.Process = true + + //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. + //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers + CorazaEvent := map[string]interface{}{ + //core rule info + "rule_type": kind, + "rule_id": rule.Rule().ID(), + //"rule_action": tx.Interruption().Action, + "rule_disruptive": rule.Disruptive(), + "rule_tags": rule.Rule().Tags(), + "rule_file": rule.Rule().File(), + "rule_file_line": rule.Rule().Line(), + "rule_revision": rule.Rule().Revision(), + "rule_secmark": rule.Rule().SecMark(), + "rule_accuracy": rule.Rule().Accuracy(), + + //http contextual infos + "upstream_addr": r.RemoteAddr, + "req_uuid": tx.ID(), + "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], + "uri": rule.URI(), + } + + if tx.Interruption() != nil { + CorazaEvent["rule_action"] = tx.Interruption().Action + } + corazaEventB, err := json.Marshal(CorazaEvent) + if err != nil { + return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) + } + evt.Line = types.Line{ + Time: time.Now(), + //should we add some info like listen addr/port/path ? + Labels: map[string]string{"type": "waf"}, + Process: true, + Module: "waf", + Src: "waf", + Raw: string(corazaEventB), + } + + return evt, nil +} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index ad72d582c26..c09312119dd 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -2,14 +2,12 @@ package wafacquisition import ( "context" - "encoding/json" "fmt" "io/ioutil" "net/http" "net/url" "os" "strings" - "time" "github.com/corazawaf/coraza/v3" corazatypes "github.com/corazawaf/coraza/v3/types" @@ -31,24 +29,59 @@ const ( OutOfBand = "outofband" ) +type WafRunner struct { + outChan chan types.Event + inChan chan ParsedRequest + inBandWaf coraza.WAF + outOfBandWaf coraza.WAF + UUID string +} + type WafSource struct { - config WafSourceConfig - logger *log.Entry - mux *http.ServeMux - server *http.Server - addr string - outChan chan types.Event - OutOfBandChan chan ParsedRequest - InBandChan chan ParsedRequest + config WafSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + InChan chan ParsedRequest inBandWaf coraza.WAF outOfBandWaf coraza.WAF + + WafRunners []WafRunner +} + +type ParsedRequest struct { + RemoteAddr string + Host string + ClientIP string + URI string + ClientHost string + Headers http.Header + URL *url.URL + Method string + Proto string + Body []byte + TransferEncoding []string + UUID string + Tx corazatypes.Transaction + ResponseChannel chan ResponseRequest +} + +type ResponseRequest struct { + ResponseChannel chan ResponseRequest + UUID string + Tx corazatypes.Transaction + Interruption *corazatypes.Interruption + Err error } type WafSourceConfig struct { ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` Path string `yaml:"path"` + WafRoutines int `yaml:"waf_routines"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -119,12 +152,17 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { ruleLoader := waf.NewWafRuleLoader() +<<<<<<< HEAD rulesCollections, err := ruleLoader.LoadWafRules() +======= + err = crowdsecWafConfig.LoadWafRules() +>>>>>>> 0c10c8f0 (update waf) if err != nil { return fmt.Errorf("cannot load WAF rules: %w", err) } +<<<<<<< HEAD var inBandRules string var outOfBandRules string @@ -143,17 +181,62 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { //w.logger.Infof("Loading rules %+v", inBandRules) fs := os.DirFS(ruleLoader.Datadir) +======= + // always have at least one waf routine + if w.config.WafRoutines == 0 { + w.config.WafRoutines = 1 + } - //in-band waf : kill on sight - inbandwaf, err := coraza.NewWAF( - coraza.NewWAFConfig(). - WithErrorCallback(logError). - WithDirectives(inBandRules).WithRootFS(fs), - ) + fs := os.DirFS(crowdsecWafConfig.Datadir) + w.InChan = make(chan ParsedRequest) + w.WafRunners = make([]WafRunner, w.config.WafRoutines) + for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { + var inBandRules string + for _, rule := range crowdsecWafConfig.InbandRules { +>>>>>>> 0c10c8f0 (update waf) - if err != nil { - return errors.Wrap(err, "Cannot create WAF") + inBandRules += rule.String() + "\n" + } + w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) + + //in-band waf : kill on sight + inbandwaf, err := coraza.NewWAF( + coraza.NewWAFConfig(). + //WithErrorCallback(logError). + WithDirectives(inBandRules).WithRootFS(fs), + ) + + if err != nil { + return errors.Wrap(err, "Cannot create WAF") + } + w.inBandWaf = inbandwaf + + var outOfBandRules string + for _, rule := range crowdsecWafConfig.OutOfBandRules { + outOfBandRules += rule.String() + "\n" + } + + w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) + //out-of-band waf : log only + outofbandwaf, err := coraza.NewWAF( + coraza.NewWAFConfig(). + //WithErrorCallback(logError). + WithDirectives(outOfBandRules).WithRootFS(fs), + ) + + if err != nil { + return errors.Wrap(err, "Cannot create WAF") + } + + runner := WafRunner{ + outOfBandWaf: outofbandwaf, + inBandWaf: inbandwaf, + inChan: w.InChan, + UUID: uuid.New().String(), + } + w.WafRunners[nbRoutine] = runner } +<<<<<<< HEAD w.inBandWaf = inbandwaf w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) @@ -168,6 +251,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf +======= + //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) + + //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec +>>>>>>> 0c10c8f0 (update waf) w.mux.HandleFunc(w.config.Path, w.wafHandler) return nil @@ -191,26 +279,17 @@ func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { w.outChan = out - w.OutOfBandChan = make(chan ParsedRequest) - w.InBandChan = make(chan ParsedRequest) t.Go(func() error { defer trace.CatchPanic("crowdsec/acquis/waf/live") - // start outOfBand GoRoutine - t.Go(func() error { - if err := w.ProcessOutBand(t); err != nil { - return errors.Wrap(err, "Processing Out of band routine failed: %s") - } - return nil - }) - - // start InBand GoRoutine - t.Go(func() error { - if err := w.ProcessInBand(t); err != nil { - return errors.Wrap(err, "Processing in-band routine failed: %s") - } - return nil - }) + w.logger.Infof("%d waf runner to start", len(w.WafRunners)) + for _, runner := range w.WafRunners { + w.logger.Infof("Running waf runner: %s", runner.UUID) + runner.outChan = out + t.Go(func() error { + return runner.Run(t) + }) + } w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) t.Go(func() error { @@ -240,22 +319,10 @@ func (w *WafSource) Dump() interface{} { return w } -type ParsedRequest struct { - RemoteAddr string - Host string - Headers http.Header - URL *url.URL - Method string - Proto string - Body []byte - TransferEncoding []string - UUID string - Tx corazatypes.Transaction -} - func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { var body []byte var err error + if r.Body != nil { body = make([]byte, 0) body, err = ioutil.ReadAll(r.Body) @@ -263,8 +330,25 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) } } + + // the real source of the request is set in 'x-client-ip' + clientIP := r.Header.Get("X-Client-Ip") + // the real target Host of the request is set in 'x-client-host' + clientHost := r.Header.Get("X-Client-Host") + // the real URI of the request is set in 'x-client-uri' + clientURI := r.Header.Get("X-Client-Uri") + + // delete those headers before coraza process the request + delete(r.Header, "x-client-ip") + delete(r.Header, "x-client-host") + delete(r.Header, "x-client-uri") + return ParsedRequest{ RemoteAddr: r.RemoteAddr, + UUID: uuid.New().String(), + ClientHost: clientHost, + ClientIP: clientIP, + URI: clientURI, Host: r.Host, Headers: r.Header, URL: r.URL, @@ -272,6 +356,7 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { Proto: r.Proto, Body: body, TransferEncoding: r.TransferEncoding, + ResponseChannel: make(chan ResponseRequest), }, nil } @@ -299,9 +384,10 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType //txx = tx.(experimental.FullTransaction) //txx.RemoveRuleByID(1) - tx.ProcessConnection(r.RemoteAddr, 0, "", 0) + tx.ProcessConnection(r.ClientIP, 0, "", 0) - tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers + //tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers + tx.ProcessURI(r.URI, r.Method, r.Proto) //FIXME: get it from the headers for k, vr := range r.Headers { for _, v := range vr { @@ -309,10 +395,10 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } } - if r.Host != "" { - tx.AddRequestHeader("Host", r.Host) + if r.ClientHost != "" { + tx.AddRequestHeader("Host", r.ClientHost) // This connector relies on the host header (now host field) to populate ServerName - tx.SetServerName(r.Host) + tx.SetServerName(r.ClientHost) } if r.TransferEncoding != nil { @@ -333,7 +419,6 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } ct := r.Headers.Get("content-type") - if r.Body != nil && len(r.Body) != 0 { it, _, err := tx.WriteRequestBody(r.Body) if err != nil { @@ -356,12 +441,10 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } } } - in, err := tx.ProcessRequestBody() if err != nil { return nil, nil, errors.Wrap(err, "Cannot process request body") } - if in != nil && wafType == InBand { return in, tx, nil } @@ -369,129 +452,57 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType return nil, tx, nil } -func (w *WafSource) TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { - evts := []types.Event{} - if r.Tx == nil { - return nil, fmt.Errorf("tx is nil") - } - for idx, rule := range r.Tx.MatchedRules() { - log.Printf("rule %d", idx) - evt, err := w.RuleMatchToEvent(rule, r.Tx, r, kind) - if err != nil { - return nil, errors.Wrap(err, "Cannot convert rule match to event") - } - evts = append(evts, evt) - } - - return evts, nil -} - -// Transforms a coraza interruption to a crowdsec event -func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) { - evt := types.Event{} - //we might want to change this based on in-band vs out-of-band ? - evt.Type = types.LOG - evt.ExpectMode = types.LIVE - //def needs fixing - evt.Stage = "s00-raw" - evt.Process = true - - //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. - //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers - CorazaEvent := map[string]interface{}{ - //core rule info - "rule_type": kind, - "rule_id": rule.Rule().ID(), - //"rule_action": tx.Interruption().Action, - "rule_disruptive": rule.Disruptive(), - "rule_tags": rule.Rule().Tags(), - "rule_file": rule.Rule().File(), - "rule_file_line": rule.Rule().Line(), - "rule_revision": rule.Rule().Revision(), - "rule_secmark": rule.Rule().SecMark(), - "rule_accuracy": rule.Rule().Accuracy(), - - //http contextual infos - "upstream_addr": r.RemoteAddr, - "req_uuid": tx.ID(), - "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], - "uri": rule.URI(), - } - - if tx.Interruption() != nil { - CorazaEvent["rule_action"] = tx.Interruption().Action - } - corazaEventB, err := json.Marshal(CorazaEvent) - if err != nil { - return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) - } - evt.Line = types.Line{ - Time: time.Now(), - //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "waf"}, - Process: true, - Module: "waf", - Src: "waf", - Raw: string(corazaEventB), - } - - return evt, nil -} - -func (w *WafSource) ProcessOutBand(t *tomb.Tomb) error { +func (r *WafRunner) Run(t *tomb.Tomb) error { for { select { case <-t.Dying(): - log.Infof("OutOfBand function is dying") + log.Infof("Waf Runner is dying") return nil - case r := <-w.OutOfBandChan: - in2, tx2, err := processReqWithEngine(w.outOfBandWaf, r, r.UUID, OutOfBand) + case request := <-r.inChan: + in, tx, err := processReqWithEngine(r.inBandWaf, request, request.UUID, InBand) + response := ResponseRequest{ + Tx: tx, + Interruption: in, + Err: err, + UUID: request.UUID, + } + // send back the result to the HTTP handler for the InBand part + request.ResponseChannel <- response + request.Tx = tx + // Generate the events for InBand channel + events, err := TxToEvents(request, InBand) + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + continue + } + + for _, evt := range events { + r.outChan <- evt + } + + // Process outBand + in, tx, err = processReqWithEngine(r.outOfBandWaf, request, request.UUID, OutOfBand) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) continue } - r.Tx = tx2 - if tx2 != nil && len(tx2.MatchedRules()) > 0 { - events, err := w.TxToEvents(r, OutOfBand) + request.Tx = tx + if tx != nil && len(tx.MatchedRules()) > 0 { + events, err := TxToEvents(request, OutOfBand) log.Infof("Request triggered by WAF, %d events to send", len(events)) for _, evt := range events { - w.outChan <- evt + r.outChan <- evt } if err != nil { log.Errorf("Cannot convert transaction to events : %s", err) continue } - log.Infof("WAF triggered : %+v", in2) - } - } - } -} - -func (w *WafSource) ProcessInBand(t *tomb.Tomb) error { - for { - select { - case <-t.Dying(): - log.Infof("OutOfBand function is dying") - return nil - case r := <-w.InBandChan: - events, err := w.TxToEvents(r, InBand) - if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue - } - - log.Infof("Request blocked by WAF, %d events to send", len(events)) - for _, evt := range events { - w.outChan <- evt } } } } func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { - //let's gen a transaction id to keep consistance accross in-band and out-of-band - uuid := uuid.New().String() - // parse the request only once parsedRequest, err := NewParsedRequestFromRequest(r) if err != nil { @@ -499,28 +510,22 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusForbidden) return } + w.InChan <- parsedRequest - //inband first - in, tx, err := processReqWithEngine(w.inBandWaf, parsedRequest, uuid, InBand) - if err != nil { //things went south - log.Errorf("Error while processing request : %s", err) - rw.WriteHeader(http.StatusForbidden) // do we want to return 403 is smth went wrong ? + message := <-parsedRequest.ResponseChannel + + if message.Err != nil { + log.Errorf("Error while processing InBAND: %s", err) + rw.WriteHeader(http.StatusOK) return } - if in != nil { + if message.Interruption != nil { rw.WriteHeader(http.StatusForbidden) - go func() { - parsedRequest.Tx = tx - w.InBandChan <- parsedRequest - }() + return } - // we finished the inband, we can return 200 rw.WriteHeader(http.StatusOK) - //now we can process out of band asynchronously - go func() { - w.OutOfBandChan <- parsedRequest - }() + return } diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index 97bfc997cb9..d004213b7ee 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -43,6 +43,16 @@ type WafConfig struct { OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` +<<<<<<< HEAD +======= + + CompiledOnLoad []CompiledHook `yaml:"-"` + CompiledPreEval []CompiledHook `yaml:"-"` + CompiledOnMatch []CompiledHook `yaml:"-"` + + MergedRules []string `yaml:"-"` + OutOfBand bool `yaml:"outofband"` +>>>>>>> 0c10c8f0 (update waf) } type WafRuleLoader struct { From 877d4fc32dea77e860c29a52e8493739a6190438 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Fri, 16 Jun 2023 14:23:53 +0200 Subject: [PATCH 023/263] update --- pkg/acquisition/modules/waf/waf.go | 36 ------------------------------ pkg/waf/waf.go | 3 --- 2 files changed, 39 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index c09312119dd..0591d9b44bd 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -152,17 +152,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { ruleLoader := waf.NewWafRuleLoader() -<<<<<<< HEAD rulesCollections, err := ruleLoader.LoadWafRules() - -======= - err = crowdsecWafConfig.LoadWafRules() ->>>>>>> 0c10c8f0 (update waf) if err != nil { return fmt.Errorf("cannot load WAF rules: %w", err) } -<<<<<<< HEAD var inBandRules string var outOfBandRules string @@ -181,22 +175,14 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { //w.logger.Infof("Loading rules %+v", inBandRules) fs := os.DirFS(ruleLoader.Datadir) -======= // always have at least one waf routine if w.config.WafRoutines == 0 { w.config.WafRoutines = 1 } - fs := os.DirFS(crowdsecWafConfig.Datadir) w.InChan = make(chan ParsedRequest) w.WafRunners = make([]WafRunner, w.config.WafRoutines) for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { - var inBandRules string - for _, rule := range crowdsecWafConfig.InbandRules { ->>>>>>> 0c10c8f0 (update waf) - - inBandRules += rule.String() + "\n" - } w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) //in-band waf : kill on sight @@ -209,13 +195,6 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { if err != nil { return errors.Wrap(err, "Cannot create WAF") } - w.inBandWaf = inbandwaf - - var outOfBandRules string - for _, rule := range crowdsecWafConfig.OutOfBandRules { - outOfBandRules += rule.String() + "\n" - } - w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) //out-of-band waf : log only outofbandwaf, err := coraza.NewWAF( @@ -236,26 +215,13 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.WafRunners[nbRoutine] = runner } -<<<<<<< HEAD - w.inBandWaf = inbandwaf w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) - //out-of-band waf : log only - outofbandwaf, err := coraza.NewWAF( - coraza.NewWAFConfig(). - WithErrorCallback(logError). - WithDirectives(outOfBandRules).WithRootFS(fs), - ) - if err != nil { return errors.Wrap(err, "Cannot create WAF") } - w.outOfBandWaf = outofbandwaf -======= - //log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec ->>>>>>> 0c10c8f0 (update waf) w.mux.HandleFunc(w.config.Path, w.wafHandler) return nil @@ -374,8 +340,6 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType tx.Close() }() - log.Infof("Processing request with %s WAF", wafType) - //this method is not exported by coraza, so we have to do it ourselves. //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ //var txx experimental.FullTransaction diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index d004213b7ee..0c61d24470c 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -43,8 +43,6 @@ type WafConfig struct { OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` -<<<<<<< HEAD -======= CompiledOnLoad []CompiledHook `yaml:"-"` CompiledPreEval []CompiledHook `yaml:"-"` @@ -52,7 +50,6 @@ type WafConfig struct { MergedRules []string `yaml:"-"` OutOfBand bool `yaml:"outofband"` ->>>>>>> 0c10c8f0 (update waf) } type WafRuleLoader struct { From 3fe6e3be14f48130cb3969f557a07b54342e00d4 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 16 Jun 2023 16:52:01 +0200 Subject: [PATCH 024/263] check for interruption and ignore empty messages --- pkg/acquisition/modules/waf/utils.go | 3 +++ pkg/acquisition/modules/waf/waf.go | 25 +++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 5e8431b2964..7aa734d5923 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -18,6 +18,9 @@ func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { } for _, rule := range r.Tx.MatchedRules() { //log.Printf("rule %d", idx) + if rule.Message() == "" { + continue + } evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) if err != nil { return nil, errors.Wrap(err, "Cannot convert rule match to event") diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 0591d9b44bd..76262607049 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -15,7 +15,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -373,9 +372,9 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType //spew.Dump(in) //spew.Dump(tx.MatchedRules()) - for _, rule := range tx.MatchedRules() { + /*for _, rule := range tx.MatchedRules() { spew.Dump(rule.Rule()) - } + }*/ //if we're inband, we should stop here, but for outofband go to the end if in != nil && wafType == InBand { @@ -432,16 +431,18 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response - request.Tx = tx - // Generate the events for InBand channel - events, err := TxToEvents(request, InBand) - if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue - } + if in != nil { + request.Tx = tx + // Generate the events for InBand channel + events, err := TxToEvents(request, InBand) + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + continue + } - for _, evt := range events { - r.outChan <- evt + for _, evt := range events { + r.outChan <- evt + } } // Process outBand From 13512891e41177323ac0a8afa4b26055f55a135a Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 4 Jul 2023 17:36:56 +0200 Subject: [PATCH 025/263] add waf_routines --- pkg/acquisition/modules/waf/utils.go | 8 +- pkg/acquisition/modules/waf/waf.go | 262 ++++++++++++++++----------- pkg/waf/env.go | 32 ++++ pkg/waf/request.go | 106 +++++++++++ pkg/waf/waf.go | 8 +- pkg/waf/waf_rules_collection.go | 2 + 6 files changed, 301 insertions(+), 117 deletions(-) create mode 100644 pkg/waf/env.go create mode 100644 pkg/waf/request.go diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 7aa734d5923..6d2932bf9e3 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -8,10 +8,12 @@ import ( corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) -func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { +func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) { evts := []types.Event{} if r.Tx == nil { return nil, fmt.Errorf("tx is nil") @@ -32,7 +34,7 @@ func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) { } // Transforms a coraza interruption to a crowdsec event -func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) { +func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -40,7 +42,7 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, //def needs fixing evt.Stage = "s00-raw" evt.Process = true - + log.Infof("SOURCE IP: %+v", rule) //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers CorazaEvent := map[string]interface{}{ diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 76262607049..e46e875ebab 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -2,14 +2,15 @@ package wafacquisition import ( "context" + "encoding/json" "fmt" - "io/ioutil" "net/http" - "net/url" "os" "strings" + "github.com/antonmedv/expr" "github.com/corazawaf/coraza/v3" + "github.com/corazawaf/coraza/v3/experimental" corazatypes "github.com/corazawaf/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -29,11 +30,20 @@ const ( ) type WafRunner struct { - outChan chan types.Event - inChan chan ParsedRequest - inBandWaf coraza.WAF - outOfBandWaf coraza.WAF - UUID string + outChan chan types.Event + inChan chan waf.ParsedRequest + inBandWaf coraza.WAF + outOfBandWaf coraza.WAF + UUID string + RulesCollections []*waf.WafRulesCollection +} + +type WafSourceConfig struct { + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + Path string `yaml:"path"` + WafRoutines int `yaml:"waf_routines"` + configuration.DataSourceCommonCfg `yaml:",inline"` } type WafSource struct { @@ -43,47 +53,15 @@ type WafSource struct { server *http.Server addr string outChan chan types.Event - InChan chan ParsedRequest + InChan chan waf.ParsedRequest - inBandWaf coraza.WAF - outOfBandWaf coraza.WAF + inBandWaf coraza.WAF + outOfBandWaf coraza.WAF + RulesCollections []*waf.WafRulesCollection WafRunners []WafRunner } -type ParsedRequest struct { - RemoteAddr string - Host string - ClientIP string - URI string - ClientHost string - Headers http.Header - URL *url.URL - Method string - Proto string - Body []byte - TransferEncoding []string - UUID string - Tx corazatypes.Transaction - ResponseChannel chan ResponseRequest -} - -type ResponseRequest struct { - ResponseChannel chan ResponseRequest - UUID string - Tx corazatypes.Transaction - Interruption *corazatypes.Interruption - Err error -} - -type WafSourceConfig struct { - ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` - Path string `yaml:"path"` - WafRoutines int `yaml:"waf_routines"` - configuration.DataSourceCommonCfg `yaml:",inline"` -} - func (w *WafSource) GetMetrics() []prometheus.Collector { return nil } @@ -156,6 +134,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return fmt.Errorf("cannot load WAF rules: %w", err) } + w.RulesCollections = rulesCollections + var inBandRules string var outOfBandRules string @@ -179,7 +159,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.config.WafRoutines = 1 } - w.InChan = make(chan ParsedRequest) + w.InChan = make(chan waf.ParsedRequest) w.WafRunners = make([]WafRunner, w.config.WafRoutines) for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) @@ -207,10 +187,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } runner := WafRunner{ - outOfBandWaf: outofbandwaf, - inBandWaf: inbandwaf, - inChan: w.InChan, - UUID: uuid.New().String(), + outOfBandWaf: outofbandwaf, + inBandWaf: inbandwaf, + inChan: w.InChan, + UUID: uuid.New().String(), + RulesCollections: rulesCollections, } w.WafRunners[nbRoutine] = runner } @@ -284,51 +265,8 @@ func (w *WafSource) Dump() interface{} { return w } -func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { - var body []byte - var err error - - if r.Body != nil { - body = make([]byte, 0) - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) - } - } - - // the real source of the request is set in 'x-client-ip' - clientIP := r.Header.Get("X-Client-Ip") - // the real target Host of the request is set in 'x-client-host' - clientHost := r.Header.Get("X-Client-Host") - // the real URI of the request is set in 'x-client-uri' - clientURI := r.Header.Get("X-Client-Uri") - - // delete those headers before coraza process the request - delete(r.Header, "x-client-ip") - delete(r.Header, "x-client-host") - delete(r.Header, "x-client-uri") - - return ParsedRequest{ - RemoteAddr: r.RemoteAddr, - UUID: uuid.New().String(), - ClientHost: clientHost, - ClientIP: clientIP, - URI: clientURI, - Host: r.Host, - Headers: r.Header, - URL: r.URL, - Method: r.Method, - Proto: r.Proto, - Body: body, - TransferEncoding: r.TransferEncoding, - ResponseChannel: make(chan ResponseRequest), - }, nil -} - -func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType string) (*corazatypes.Interruption, corazatypes.Transaction, error) { +func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) { var in *corazatypes.Interruption - tx := waf.NewTransactionWithID(uuid) - if tx.IsRuleEngineOff() { log.Printf("engine is off") return nil, nil, nil @@ -346,7 +284,6 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType //txx := experimental.ToFullInterface(tx) //txx = tx.(experimental.FullTransaction) //txx.RemoveRuleByID(1) - tx.ProcessConnection(r.ClientIP, 0, "", 0) //tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers @@ -369,12 +306,15 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType } in = tx.ProcessRequestHeaders() + //spew.Dump(in) //spew.Dump(tx.MatchedRules()) - /*for _, rule := range tx.MatchedRules() { - spew.Dump(rule.Rule()) - }*/ + for _, rule := range tx.MatchedRules() { + if rule.Message() == "" { + continue + } + } //if we're inband, we should stop here, but for outofband go to the end if in != nil && wafType == InBand { @@ -422,17 +362,105 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { log.Infof("Waf Runner is dying") return nil case request := <-r.inChan: - in, tx, err := processReqWithEngine(r.inBandWaf, request, request.UUID, InBand) - response := ResponseRequest{ - Tx: tx, - Interruption: in, - Err: err, - UUID: request.UUID, + inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID) + expTx := inBoundTx.(experimental.FullTransaction) + // we use this internal transaction for the expr helpers + tx := waf.NewTransaction(expTx) + + //Run the pre_eval hooks + for _, rules := range r.RulesCollections { + if len(rules.CompiledPreEval) == 0 { + continue + } + for _, compiledHook := range rules.CompiledPreEval { + if compiledHook.Filter != nil { + res, err := expr.Run(compiledHook.Filter, map[string]interface{}{ + "rules": rules, + "req": request, + }) + if err != nil { + log.Errorf("unable to run PreEval filter: %s", err) + continue + } + + switch t := res.(type) { + case bool: + if t == false { + log.Infof("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range compiledHook.Apply { + _, err := expr.Run(applyExpr, map[string]interface{}{ + "rules": rules, + "req": request, + "RemoveRuleByID": tx.RemoveRuleByIDWithError, + }) + if err != nil { + log.Errorf("unable to apply filter: %s", err) + continue + } + } + } + } + + in, expTx, err := processReqWithEngine(expTx, request, InBand) + request.Tx = expTx + + response := waf.NewResponseRequest(expTx, in, request.UUID, err) + + // run the on_match hooks + for _, rules := range r.RulesCollections { + if len(rules.CompiledOnMatch) == 0 { + continue + } + for _, compiledHook := range rules.CompiledOnMatch { + if compiledHook.Filter != nil { + res, err := expr.Run(compiledHook.Filter, map[string]interface{}{ + "rules": rules, + "req": request, + }) + if err != nil { + log.Errorf("unable to run PreEval filter: %s", err) + continue + } + + switch t := res.(type) { + case bool: + if t == false { + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range compiledHook.Apply { + _, err := expr.Run(applyExpr, map[string]interface{}{ + "rules": rules, + "req": request, + "RemoveRuleByID": tx.RemoveRuleByIDWithError, + "SetRemediation": response.SetRemediation, + "SetRemediationByID": response.SetRemediationByID, + "CancelEvent": response.CancelEvent, + }) + if err != nil { + log.Errorf("unable to apply filter: %s", err) + continue + } + } + } } + // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response - if in != nil { - request.Tx = tx + if in != nil && response.SendEvents { // Generate the events for InBand channel events, err := TxToEvents(request, InBand) if err != nil { @@ -446,13 +474,15 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } // Process outBand - in, tx, err = processReqWithEngine(r.outOfBandWaf, request, request.UUID, OutOfBand) + outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) + expTx = outBandTx.(experimental.FullTransaction) + in, expTx, err = processReqWithEngine(expTx, request, OutOfBand) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) continue } - request.Tx = tx - if tx != nil && len(tx.MatchedRules()) > 0 { + request.Tx = expTx + if expTx != nil && len(expTx.MatchedRules()) > 0 { events, err := TxToEvents(request, OutOfBand) log.Infof("Request triggered by WAF, %d events to send", len(events)) for _, evt := range events { @@ -467,9 +497,13 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } +type BodyResponse struct { + Action string `json:"action"` +} + func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { // parse the request only once - parsedRequest, err := NewParsedRequestFromRequest(r) + parsedRequest, err := waf.NewParsedRequestFromRequest(r) if err != nil { log.Errorf("%s", err) rw.WriteHeader(http.StatusForbidden) @@ -487,10 +521,22 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { if message.Interruption != nil { rw.WriteHeader(http.StatusForbidden) + body, err := json.Marshal(BodyResponse{Action: message.Interruption.Action}) + if err != nil { + log.Errorf("unable to build response: %s", err) + } else { + rw.Write(body) + } return } rw.WriteHeader(http.StatusOK) + body, err := json.Marshal(BodyResponse{Action: "allow"}) + if err != nil { + log.Errorf("unable to build response: %s", err) + } else { + rw.Write(body) + } return } diff --git a/pkg/waf/env.go b/pkg/waf/env.go new file mode 100644 index 00000000000..797fe5c37a6 --- /dev/null +++ b/pkg/waf/env.go @@ -0,0 +1,32 @@ +package waf + +import "github.com/corazawaf/coraza/v3/experimental" + +type Transaction struct { + Tx experimental.FullTransaction +} + +func NewTransaction(tx experimental.FullTransaction) Transaction { + return Transaction{Tx: tx} +} + +func (t *Transaction) RemoveRuleByIDWithError(id int) error { + t.Tx.RemoveRuleByID(id) + return nil +} + +func GetEnv() map[string]interface{} { + ResponseRequest := ResponseRequest{} + ParsedRequest := ParsedRequest{} + Rules := &WafRulesCollection{} + Tx := Transaction{} + + return map[string]interface{}{ + "rules": Rules, + "req": ParsedRequest, + "SetRemediation": ResponseRequest.SetRemediation, + "SetRemediationByID": ResponseRequest.SetRemediationByID, + "CancelEvent": ResponseRequest.CancelEvent, + "RemoveRuleByID": Tx.RemoveRuleByIDWithError, + } +} diff --git a/pkg/waf/request.go b/pkg/waf/request.go new file mode 100644 index 00000000000..29d9ab35f33 --- /dev/null +++ b/pkg/waf/request.go @@ -0,0 +1,106 @@ +package waf + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" + + "github.com/corazawaf/coraza/v3/experimental" + corazatypes "github.com/corazawaf/coraza/v3/types" + "github.com/google/uuid" +) + +type ResponseRequest struct { + UUID string + Tx corazatypes.Transaction + Interruption *corazatypes.Interruption + Err error + SendEvents bool +} + +func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest { + return ResponseRequest{ + UUID: UUID, + Tx: Tx, + Interruption: in, + Err: err, + SendEvents: true, + } +} + +func (r *ResponseRequest) SetRemediation(remediation string) error { + r.Interruption.Action = remediation + return nil +} + +func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { + if r.Interruption.RuleID == ID { + r.Interruption.Action = remediation + } + return nil +} + +func (r *ResponseRequest) CancelEvent() error { + // true by default + r.SendEvents = false + return nil +} + +type ParsedRequest struct { + RemoteAddr string + Host string + ClientIP string + URI string + ClientHost string + Headers http.Header + URL *url.URL + Method string + Proto string + Body []byte + TransferEncoding []string + UUID string + Tx experimental.FullTransaction + ResponseChannel chan ResponseRequest +} + +func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { + var body []byte + var err error + + if r.Body != nil { + body = make([]byte, 0) + body, err = ioutil.ReadAll(r.Body) + if err != nil { + return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) + } + } + + // the real source of the request is set in 'x-client-ip' + clientIP := r.Header.Get("X-Client-Ip") + // the real target Host of the request is set in 'x-client-host' + clientHost := r.Header.Get("X-Client-Host") + // the real URI of the request is set in 'x-client-uri' + clientURI := r.Header.Get("X-Client-Uri") + + // delete those headers before coraza process the request + delete(r.Header, "x-client-ip") + delete(r.Header, "x-client-host") + delete(r.Header, "x-client-uri") + + return ParsedRequest{ + RemoteAddr: r.RemoteAddr, + UUID: uuid.New().String(), + ClientHost: clientHost, + ClientIP: clientIP, + URI: clientURI, + Host: r.Host, + Headers: r.Header, + URL: r.URL, + Method: r.Method, + Proto: r.Proto, + Body: body, + TransferEncoding: r.TransferEncoding, + ResponseChannel: make(chan ResponseRequest), + }, nil +} diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index 0c61d24470c..d2d4d0dc8eb 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -67,9 +67,7 @@ func buildHook(hook Hook) (CompiledHook, error) { compiledHook.Filter = program } for _, apply := range hook.Apply { - program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{ - "rules": &WafRulesCollection{}, - })...) + program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...) if err != nil { return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err) } @@ -173,9 +171,7 @@ func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { //Ignore filter for on load ? if onLoadHook.Apply != nil { for exprIdx, applyExpr := range onLoadHook.Apply { - _, err := expr.Run(applyExpr, map[string]interface{}{ - "rules": collection, - }) + _, err := expr.Run(applyExpr, GetEnv()) if err != nil { w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) continue diff --git a/pkg/waf/waf_rules_collection.go b/pkg/waf/waf_rules_collection.go index 3ce130e5645..f6669527f9e 100644 --- a/pkg/waf/waf_rules_collection.go +++ b/pkg/waf/waf_rules_collection.go @@ -4,6 +4,8 @@ import "strings" type WafRule struct { RawRule string + RuleID string + InBand bool } // This is the "compiled" state of a WafConfig From 84b6570554f4f5a8e7d2659a947a4375a98ce203 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 4 Jul 2023 18:46:20 +0200 Subject: [PATCH 026/263] Revert "Merge remote-tracking branch 'origin' into coraza_poc_acquis" This reverts commit 7098e971c7f25682973606291731a68d557759cc, reversing changes made to 13512891e41177323ac0a8afa4b26055f55a135a. --- .github/workflows/bats-hub.yml | 28 +- .github/workflows/bats-mysql.yml | 28 +- .github/workflows/bats-postgres.yml | 28 +- .github/workflows/bats-sqlite-coverage.yml | 28 +- .github/workflows/cache-cleanup.yaml | 35 - .github/workflows/ci-windows-build-msi.yml | 23 +- .github/workflows/docker-tests.yml | 12 + .github/workflows/go-tests-windows.yml | 23 +- .github/workflows/go-tests.yml | 31 +- .github/workflows/release_publish-package.yml | 27 +- .gitignore | 9 +- .golangci.yml | 2 +- Dockerfile | 20 +- Dockerfile.debian | 30 +- Makefile | 99 +- azure-pipelines.yml | 4 +- cmd/crowdsec-cli/alerts.go | 8 +- cmd/crowdsec-cli/capi.go | 19 +- cmd/crowdsec-cli/config_backup.go | 42 +- cmd/crowdsec-cli/config_restore.go | 24 +- cmd/crowdsec-cli/copyfile.go | 73 -- cmd/crowdsec-cli/decisions.go | 245 ++++- cmd/crowdsec-cli/decisions_import.go | 272 ----- cmd/crowdsec-cli/lapi.go | 6 +- cmd/crowdsec-cli/machines.go | 33 +- cmd/crowdsec-cli/machines_table.go | 6 +- cmd/crowdsec-cli/notifications.go | 58 +- cmd/crowdsec-cli/scenarios.go | 3 +- cmd/crowdsec-cli/support.go | 11 +- cmd/crowdsec-cli/utils.go | 6 +- cmd/crowdsec/api.go | 11 +- cmd/crowdsec/crowdsec.go | 19 +- cmd/crowdsec/main.go | 46 +- cmd/crowdsec/output.go | 15 +- cmd/crowdsec/pour.go | 4 +- cmd/crowdsec/run_in_svc_windows.go | 13 +- cmd/crowdsec/serve.go | 35 +- cmd/crowdsec/win_service.go | 4 +- debian/control | 4 +- debian/rules | 12 +- debian/templates | 2 +- docker/docker_start.sh | 2 +- docker/test/tests/test_hub_collections.py | 32 +- go.mod | 89 +- go.sum | 85 +- mk/__gmsl | 969 ------------------ mk/gmsl | 85 -- mk/gmsl.html | 733 ------------- mk/platform/unix_common.mk | 8 +- mk/platform/windows.mk | 8 +- pkg/acquisition/acquisition.go | 46 +- pkg/acquisition/acquisition_test.go | 2 +- .../modules/cloudwatch/cloudwatch.go | 16 +- pkg/acquisition/modules/docker/docker.go | 7 +- pkg/acquisition/modules/file/file.go | 22 +- pkg/acquisition/modules/file/file_test.go | 4 +- .../modules/journalctl/journalctl.go | 3 +- pkg/acquisition/modules/kafka/kafka.go | 11 +- pkg/acquisition/modules/kafka/kafka_test.go | 13 +- pkg/acquisition/modules/kinesis/kinesis.go | 37 +- .../modules/kubernetesaudit/k8s_audit.go | 14 +- pkg/acquisition/modules/s3/s3.go | 13 +- .../syslog/internal/server/syslogserver.go | 9 +- pkg/acquisition/modules/syslog/syslog.go | 5 +- pkg/apiclient/alerts_service.go | 10 +- pkg/apiclient/alerts_service_test.go | 7 +- pkg/apiclient/auth.go | 26 +- pkg/apiclient/auth_service.go | 2 + pkg/apiclient/auth_service_test.go | 5 +- pkg/apiclient/client.go | 7 +- pkg/apiclient/client_http_test.go | 4 +- pkg/apiclient/client_test.go | 3 +- pkg/apiclient/decisions_service.go | 7 +- pkg/apiclient/decisions_service_test.go | 7 +- pkg/apiclient/decisions_sync_service.go | 8 +- pkg/apiclient/signal.go | 5 +- pkg/apiserver/apic.go | 84 +- pkg/apiserver/apic_metrics.go | 145 --- pkg/apiserver/apic_metrics_test.go | 101 -- pkg/apiserver/apic_test.go | 84 ++ pkg/apiserver/apiserver.go | 28 +- pkg/apiserver/controllers/v1/controller.go | 6 +- pkg/apiserver/middlewares/v1/api_key.go | 10 +- pkg/apiserver/middlewares/v1/jwt.go | 2 +- pkg/apiserver/middlewares/v1/tls_auth.go | 7 +- pkg/csconfig/common.go | 3 +- pkg/csconfig/config_paths.go | 4 +- pkg/csconfig/profiles.go | 4 +- pkg/csplugin/broker.go | 12 +- pkg/csplugin/broker_suite_test.go | 18 +- pkg/csplugin/broker_test.go | 19 +- pkg/csplugin/broker_win_test.go | 2 +- pkg/csplugin/listfiles.go | 3 +- pkg/csplugin/listfiles_test.go | 10 +- pkg/csplugin/notifier.go | 3 +- pkg/csplugin/utils.go | 11 +- pkg/csplugin/utils_windows.go | 35 +- pkg/csplugin/watcher.go | 3 +- pkg/csplugin/watcher_test.go | 17 +- pkg/csprofiles/csprofiles.go | 9 +- pkg/csprofiles/csprofiles_test.go | 7 +- pkg/cticlient/example/fire.go | 3 - pkg/cticlient/types.go | 2 +- pkg/cwhub/cwhub.go | 13 +- pkg/cwhub/cwhub_test.go | 18 +- pkg/cwhub/download.go | 5 +- pkg/cwhub/helpers.go | 14 +- pkg/cwhub/helpers_test.go | 6 +- pkg/cwhub/loader.go | 7 +- pkg/database/alerts.go | 35 +- pkg/database/bouncers.go | 3 +- pkg/database/config.go | 3 +- pkg/database/decisions.go | 6 +- pkg/database/machines.go | 4 +- pkg/exprhelpers/jsonextract.go | 4 +- pkg/hubtest/hubtest.go | 3 +- pkg/hubtest/hubtest_item.go | 3 +- pkg/hubtest/parser_assert.go | 8 +- pkg/hubtest/scenario_assert.go | 10 +- pkg/leakybucket/README.md | 120 +-- pkg/leakybucket/bayesian.go | 163 --- pkg/leakybucket/bucket.go | 4 - pkg/leakybucket/manager_load.go | 29 - pkg/leakybucket/manager_load_test.go | 22 - pkg/leakybucket/overflows.go | 19 +- .../guillotine-bayesian-bucket/bucket.yaml | 21 - .../guillotine-bayesian-bucket/scenarios.yaml | 1 - .../guillotine-bayesian-bucket/test.json | 50 - .../multiple-bayesian-bucket/bucket.yaml | 21 - .../multiple-bayesian-bucket/scenarios.yaml | 1 - .../tests/multiple-bayesian-bucket/test.json | 64 -- .../tests/simple-bayesian-bucket/bucket.yaml | 19 - .../simple-bayesian-bucket/scenarios.yaml | 1 - .../tests/simple-bayesian-bucket/test.json | 50 - pkg/metabase/api.go | 10 +- pkg/metabase/database.go | 9 +- pkg/metabase/metabase.go | 40 +- pkg/parser/enrich.go | 5 +- pkg/parser/enrich_date.go | 3 +- pkg/parser/enrich_dns.go | 4 +- pkg/parser/enrich_geoip.go | 6 +- pkg/parser/enrich_unmarshal.go | 3 +- pkg/parser/node.go | 139 +-- pkg/parser/node_test.go | 18 +- pkg/parser/parsing_test.go | 12 +- pkg/parser/runtime.go | 32 +- pkg/parser/stage.go | 17 +- pkg/parser/unix_parser.go | 11 +- pkg/parser/whitelist.go | 1 - pkg/setup/detect.go | 7 +- pkg/setup/detect_test.go | 13 +- pkg/{cwhub => types}/dataset.go | 20 +- pkg/{cwhub => types}/dataset_test.go | 4 +- pkg/types/datasource.go | 16 - pkg/{parser => types}/grok_pattern.go | 3 +- pkg/types/ip.go | 13 +- pkg/types/utils.go | 98 +- plugins/notifications/dummy/Makefile | 5 + plugins/notifications/email/Makefile | 5 + plugins/notifications/email/go.mod | 12 +- plugins/notifications/email/go.sum | 26 +- plugins/notifications/http/Makefile | 5 + plugins/notifications/http/go.mod | 12 +- plugins/notifications/http/go.sum | 26 +- plugins/notifications/slack/Makefile | 5 + plugins/notifications/slack/go.mod | 12 +- plugins/notifications/slack/go.sum | 26 +- plugins/notifications/splunk/Makefile | 5 + plugins/notifications/splunk/go.mod | 12 +- plugins/notifications/splunk/go.sum | 26 +- rpm/SPECS/crowdsec.spec | 4 + test/README.md | 10 +- test/ansible/debug_tools.yml | 2 - test/ansible/provision_dependencies.yml | 2 - test/ansible/requirements.yml | 4 +- .../ansible/roles/make_fixture/tasks/main.yml | 4 +- test/ansible/vagrant/alma-8/Vagrantfile | 4 - test/ansible/vagrant/alma-9/Vagrantfile | 4 - test/ansible/vagrant/centos-7/Vagrantfile | 2 - test/ansible/vagrant/centos-8/Vagrantfile | 6 +- test/ansible/vagrant/centos-9/Vagrantfile | 4 - test/ansible/vagrant/common | 65 +- .../vagrant/debian-10-buster/Vagrantfile | 2 - .../vagrant/debian-11-bullseye/Vagrantfile | 2 - .../vagrant/debian-12-bookworm/Vagrantfile | 11 - .../vagrant/debian-9-stretch/Vagrantfile | 3 - .../vagrant/debian-testing/Vagrantfile | 2 - .../experimental/hardenedbsd-13/Vagrantfile | 21 +- .../experimental/hardenedbsd-13/bootstrap | 5 + .../experimental/openbsd-6/Vagrantfile | 14 - .../vagrant/experimental/openbsd-6/skip | 9 - .../experimental/openbsd-7/Vagrantfile | 24 +- .../vagrant/experimental/openbsd-7/bootstrap | 6 + test/ansible/vagrant/fedora-33/Vagrantfile | 2 - test/ansible/vagrant/fedora-34/Vagrantfile | 2 - test/ansible/vagrant/fedora-35/Vagrantfile | 2 - test/ansible/vagrant/fedora-36/Vagrantfile | 2 - test/ansible/vagrant/fedora-37/Vagrantfile | 11 - test/ansible/vagrant/fedora-38/Vagrantfile | 10 - test/ansible/vagrant/freebsd-12/Vagrantfile | 3 - test/ansible/vagrant/freebsd-13/Vagrantfile | 3 - test/ansible/vagrant/oracle-7/Vagrantfile | 3 - test/ansible/vagrant/oracle-8/Vagrantfile | 3 - test/ansible/vagrant/oracle-9/Vagrantfile | 3 - test/ansible/vagrant/rocky-8/Vagrantfile | 4 - test/ansible/vagrant/rocky-9/Vagrantfile | 4 - .../vagrant/ubuntu-16.04-xenial/Vagrantfile | 2 - .../vagrant/ubuntu-18.04-bionic/Vagrantfile | 2 - .../vagrant/ubuntu-20.04-focal/Vagrantfile | 3 - .../vagrant/ubuntu-22.04-jammy/Vagrantfile | 2 - .../vagrant/ubuntu-22.10-kinetic/Vagrantfile | 10 - .../vagrant/ubuntu-23.04-lunar/Vagrantfile | 10 - .../vagrant/wizard/centos-8/Vagrantfile | 43 +- .../ansible/vagrant/wizard/centos-8/bootstrap | 5 + test/ansible/vagrant/wizard/common | 67 -- .../wizard/debian-10-buster/Vagrantfile | 12 - .../wizard/debian-11-bullseye/Vagrantfile | 12 - .../wizard/debian-12-bookworm/Vagrantfile | 12 - .../wizard/debian-bullseye/Vagrantfile | 43 + .../vagrant/wizard/debian-bullseye/bootstrap | 5 + .../vagrant/wizard/debian-buster/Vagrantfile | 43 + .../vagrant/wizard/debian-buster/bootstrap | 5 + .../vagrant/wizard/fedora-36/Vagrantfile | 39 +- .../vagrant/wizard/fedora-36/bootstrap | 5 + .../wizard/ubuntu-22.04-jammy/Vagrantfile | 44 +- .../wizard/ubuntu-22.04-jammy/bootstrap | 5 + .../wizard/ubuntu-22.10-kinetic/Vagrantfile | 44 +- test/ansible/vars/go.yml | 2 +- test/bats-detect/openresty-deb.bats | 7 +- test/bats-detect/openresty-rpm.bats | 8 +- test/bats.mk | 3 +- test/bats/01_crowdsec.bats | 50 +- test/bats/01_cscli.bats | 8 +- test/bats/04_capi.bats | 2 +- test/bats/72_plugin_badconfig.bats | 18 +- test/bats/90_decisions.bats | 121 +-- test/bats/testdata/90_decisions/csv_decisions | 6 - test/bats/testdata/90_decisions/decisions.csv | 6 - .../bats/testdata/90_decisions/decisions.json | 42 - .../bats/testdata/90_decisions/json_decisions | 42 - test/bin/assert-crowdsec-not-running | 11 +- test/bin/check-requirements | 34 +- test/lib/setup_file.sh | 18 - test/tools/.do-not-remove | 1 - 244 files changed, 2061 insertions(+), 4715 deletions(-) delete mode 100644 .github/workflows/cache-cleanup.yaml delete mode 100644 cmd/crowdsec-cli/copyfile.go delete mode 100644 cmd/crowdsec-cli/decisions_import.go delete mode 100644 mk/__gmsl delete mode 100644 mk/gmsl delete mode 100644 mk/gmsl.html delete mode 100644 pkg/apiserver/apic_metrics.go delete mode 100644 pkg/apiserver/apic_metrics_test.go delete mode 100644 pkg/leakybucket/bayesian.go delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/test.json delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml delete mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/test.json rename pkg/{cwhub => types}/dataset.go (70%) rename pkg/{cwhub => types}/dataset_test.go (94%) delete mode 100644 pkg/types/datasource.go rename pkg/{parser => types}/grok_pattern.go (99%) delete mode 100644 test/ansible/vagrant/debian-12-bookworm/Vagrantfile create mode 100755 test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap delete mode 100644 test/ansible/vagrant/experimental/openbsd-6/Vagrantfile delete mode 100755 test/ansible/vagrant/experimental/openbsd-6/skip create mode 100755 test/ansible/vagrant/experimental/openbsd-7/bootstrap delete mode 100644 test/ansible/vagrant/fedora-37/Vagrantfile delete mode 100644 test/ansible/vagrant/fedora-38/Vagrantfile delete mode 100644 test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile delete mode 100644 test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/centos-8/bootstrap delete mode 100644 test/ansible/vagrant/wizard/common delete mode 100644 test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile delete mode 100644 test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile delete mode 100644 test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile create mode 100644 test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/debian-bullseye/bootstrap create mode 100644 test/ansible/vagrant/wizard/debian-buster/Vagrantfile create mode 100755 test/ansible/vagrant/wizard/debian-buster/bootstrap create mode 100755 test/ansible/vagrant/wizard/fedora-36/bootstrap create mode 100755 test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap delete mode 100644 test/bats/testdata/90_decisions/csv_decisions delete mode 100644 test/bats/testdata/90_decisions/decisions.csv delete mode 100644 test/bats/testdata/90_decisions/decisions.json delete mode 100644 test/bats/testdata/90_decisions/json_decisions delete mode 100644 test/tools/.do-not-remove diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index c62229cbe3c..026e8feaac2 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -27,26 +27,40 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 + run: make bats-clean bats-build bats-fixture - name: "Run hub tests" run: make bats-test-hub diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml index 529c3f52162..3c8825a47d9 100644 --- a/.github/workflows/bats-mysql.yml +++ b/.github/workflows/bats-mysql.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -34,27 +34,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture env: DB_BACKEND: mysql MYSQL_HOST: 127.0.0.1 diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml index 91e7ac10361..de99a8e22d3 100644 --- a/.github/workflows/bats-postgres.yml +++ b/.github/workflows/bats-postgres.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -35,27 +35,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture (DB_BACKEND: pgx)" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture env: DB_BACKEND: pgx PGHOST: 127.0.0.1 diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index 17b2aac0989..7b2e763b32d 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -24,27 +24,41 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd + go install github.com/mikefarah/yq/v4@latest + go install github.com/cloudflare/cfssl/cmd/cfssl@master + go install github.com/cloudflare/cfssl/cmd/cfssljson@master - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture BUILD_STATIC=1 + make clean bats-build bats-fixture - name: "Run tests" run: make bats-test diff --git a/.github/workflows/cache-cleanup.yaml b/.github/workflows/cache-cleanup.yaml deleted file mode 100644 index d193650246b..00000000000 --- a/.github/workflows/cache-cleanup.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches - -name: cleanup caches by a branch -on: - pull_request: - types: - - closed - -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Cleanup - run: | - gh extension install actions/gh-actions-cache - - REPO=${{ github.repository }} - BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" - - echo "Fetching list of cache key" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) - - ## Setting this to not fail the workflow while deleting cache keys. - set +e - echo "Deleting caches..." - for cacheKey in $cacheKeysForPR - do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm - done - echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml index 3b66b0c1fe3..b9d9722f3d7 100644 --- a/.github/workflows/ci-windows-build-msi.yml +++ b/.github/workflows/ci-windows-build-msi.yml @@ -23,27 +23,38 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: Build runs-on: windows-2019 steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build - run: make windows_installer BUILD_RE2_WASM=1 + run: make windows_installer - name: Upload MSI uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 913c4766238..6476d9f0aa1 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -30,6 +30,18 @@ jobs: with: config: .github/buildkit.toml + # - name: "Build flavor: full" + # uses: docker/build-push-action@v4 + # with: + # context: . + # file: ./Dockerfile + # tags: crowdsecurity/crowdsec:test + # target: full + # platforms: linux/amd64 + # load: true + # cache-from: type=gha + # cache-to: type=gha,mode=min + - name: "Build flavor: slim" uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 500fc58ce99..b8e81bbe083 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -22,28 +22,39 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: windows-2022 steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build run: | - make build BUILD_RE2_WASM=1 + make build - name: Run tests run: | diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 079f6c827c8..9b4adb4ce69 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -34,7 +34,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: "Build + tests" runs-on: ubuntu-latest @@ -110,30 +110,35 @@ jobs: steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - - name: Build and run tests, static + - name: Build and run tests run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev go install github.com/ory/go-acc@v0.2.8 go install github.com/kyoh86/richgo@v0.3.10 set -o pipefail - make build BUILD_STATIC=1 - make go-acc | richgo testfilter - - - name: Run tests again, dynamic - run: | - make clean build + make build make go-acc | richgo testfilter - name: Upload unit coverage to Codecov diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml index 54419cc6e51..60035573958 100644 --- a/.github/workflows/release_publish-package.yml +++ b/.github/workflows/release_publish-package.yml @@ -14,32 +14,41 @@ jobs: build: strategy: matrix: - go-version: ["1.20.5"] + go-version: ["1.20.4"] name: Build and upload binary package runs-on: ubuntu-latest steps: + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 + - name: Cache Go modules + uses: actions/cache@v3 with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.go-version }}-go- - name: Build the binaries - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev - make vendor release BUILD_STATIC=1 + run: make release - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | tag_name="${GITHUB_REF##*/}" - hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name" + hub release edit -a crowdsec-release.tgz -m "" "$tag_name" diff --git a/.gitignore b/.gitignore index 8fe1778baec..4c5cb0a1ffa 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,6 @@ *.test *.cover -# Test dependencies -test/tools/* - # VMs used for dev/test .vagrant @@ -33,10 +30,8 @@ test/coverage/* *.swp *.swo -# Dependencies are not vendored by default, but a tarball is created by "make vendor" -# and provided in the release. Used by freebsd, gentoo, etc. -vendor/ -vendor.tgz +# Dependency directories (remove the comment below to include it) +# vendor/ # crowdsec binaries cmd/crowdsec-cli/cscli diff --git a/.golangci.yml b/.golangci.yml index faa67c4bb80..79900ae7df1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -65,6 +65,7 @@ linters: # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers # - bidichk # Checks for dangerous unicode character sequences # - decorder # check declaration order and count of types, constants, variables and functions + # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases @@ -124,7 +125,6 @@ linters: - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers - wastedassign # wastedassign finds wasted assignment statements. - wrapcheck # Checks that errors returned from external packages are wrapped - - depguard # Go linter that checks if package imports are in a list of acceptable packages # # Recommended? (requires some work) diff --git a/Dockerfile b/Dockerfile index 2073d3c5a54..da1c3ab06a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,33 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.5 +ARG GOVERSION=1.20.4 FROM golang:${GOVERSION}-alpine AS build WORKDIR /go/src/crowdsec -# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway. +COPY . . + +# Alpine does not ship a static version of re2, we can build it ourselves +# Later versions require 'abseil', which is likewise not available in its static form ENV RE2_VERSION=2023-03-01 # wizard.sh requires GNU coreutils -RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ tar -xzf ${RE2_VERSION}.tar.gz && \ cd re2-${RE2_VERSION} && \ + make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.34.1 - -COPY . . - -RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ + cd - && \ + make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists + cscli parsers install crowdsecurity/whitelists && \ + go install github.com/mikefarah/yq/v4@v4.31.2 # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete diff --git a/Dockerfile.debian b/Dockerfile.debian index 61778cd9eb4..10b06befdf0 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,41 +1,32 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.5 +ARG GOVERSION=1.20.4 -FROM golang:${GOVERSION}-bookworm AS build +FROM golang:${GOVERSION}-bullseye AS build WORKDIR /go/src/crowdsec +COPY . . + ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" -# We like to choose the release of re2 to use, the debian version is usually older. -ENV RE2_VERSION=2023-03-01 - # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ - wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ - tar -xzf ${RE2_VERSION}.tar.gz && \ - cd re2-${RE2_VERSION} && \ - make && \ - make install && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.34.1 - -COPY . . - -RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ + make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists + cscli parsers install crowdsecurity/whitelists && \ + go install github.com/mikefarah/yq/v4@v4.31.2 # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete -FROM debian:bookworm-slim as slim +FROM debian:bullseye-slim as slim ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" @@ -53,6 +44,9 @@ RUN apt-get update && \ mkdir -p /staging/var/lib/crowdsec && \ mkdir -p /var/lib/crowdsec/data +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \ + && apt-get update && apt-get install -t bullseye-backports -y libsystemd0 + COPY --from=build /go/bin/yq /usr/local/bin/yq COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec diff --git a/Makefile b/Makefile index 0fb36261ffe..d6f1b95f237 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,8 @@ include mk/platform.mk -include mk/gmsl -# By default, this build requires the C++ re2 library to be installed. -# -# Debian/Ubuntu: apt install libre2-dev -# Fedora/CentOS: dnf install re2-devel -# FreeBSD: pkg install re2 -# Alpine: apk add re2-dev -# Windows: choco install re2 -# MacOS: brew install re2 - -# To build without re2, run "make BUILD_RE2_WASM=1" -# The WASM version is slower and introduces a short delay when starting a process -# (including cscli) so it is not recommended for production use. -BUILD_RE2_WASM ?= 0 - -# To build static binaries, run "make BUILD_STATIC=1". -# On some platforms, this requires additional packages -# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository). -# If the static build fails at the link stage, it might be because the static library is not provided -# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it. -BUILD_STATIC ?= 0 - -# List of plugins to build -PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) - -# Can be overriden, if you can deal with the consequences BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 -#-------------------------------------- - GOCMD = go GOTEST = $(GOCMD) test @@ -38,6 +10,8 @@ BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec CSCLI_FOLDER = ./cmd/crowdsec-cli/ + +PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) @@ -48,14 +22,8 @@ RELDIR = crowdsec-$(BUILD_VERSION) GO_MODULE_NAME = github.com/crowdsecurity/crowdsec -# Check if a given value is considered truthy and returns "0" or "1". -# A truthy value is one of the following: "1", "yes", or "true", case-insensitive. -# -# Usage: -# ifeq ($(call bool,$(FOO)),1) -# $(info Let's foo) -# endif -bool = $(if $(filter $(call lc, $1),1 yes true),1,0) +# see if we have libre2-dev installed for C++ optimizations +RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) #-------------------------------------- # @@ -78,30 +46,13 @@ endif GO_TAGS := netgo,osusergo,sqlite_omit_load_extension -ifeq ($(call bool,$(BUILD_RE2_WASM)),0) -ifeq ($(PKG_CONFIG),) - $(error "pkg-config is not available. Please install pkg-config.") -endif - -ifeq ($(RE2_CHECK),) -# we could detect the platform and suggest the command to install -RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version" -else +ifneq (,$(RE2_CHECK)) # += adds a space that we don't want GO_TAGS := $(GO_TAGS),re2_cgo LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif -endif - -ifeq ($(call bool,$(BUILD_STATIC)),1) -BUILD_TYPE = static -EXTLDFLAGS := -extldflags '-static' -else -BUILD_TYPE = dynamic -EXTLDFLAGS := -endif -export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ +export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) @@ -113,15 +64,12 @@ endif .PHONY: build build: pre-build goversion crowdsec cscli plugins -# Sanity checks and build information .PHONY: pre-build pre-build: - $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH)) - -ifneq (,$(RE2_FAIL)) - $(error $(RE2_FAIL)) +ifdef BUILD_STATIC + $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) endif - + $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) ifneq (,$(RE2_CHECK)) $(info Using C++ regexp library) else @@ -165,7 +113,6 @@ testclean: bats-clean @$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR) @$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR) -# for the tests with localstack export AWS_ENDPOINT_FORCE=http://localhost:4566 export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -173,18 +120,15 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY testenv: @echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)' -# run the tests with localstack .PHONY: test test: testenv goversion $(GOTEST) $(LD_OPTS) ./... -# run the tests with localstack and coverage .PHONY: go-acc go-acc: testenv goversion go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \ sed 's/ *coverage:.*of statements in.*//' -# mock AWS services .PHONY: localstack localstack: docker-compose -f test/localstack/docker-compose.yml up @@ -193,27 +137,13 @@ localstack: localstack-stop: docker-compose -f test/localstack/docker-compose.yml down -# list of plugins that contain go.mod -PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi)) - -# build vendor.tgz to be distributed with the release .PHONY: vendor vendor: - $(foreach plugin_dir,$(PLUGIN_VENDOR), \ - cd $(plugin_dir) >/dev/null && \ - $(GOCMD) mod vendor && \ - cd - >/dev/null; \ - ) - $(GOCMD) mod vendor - tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor) - -# remove vendor directories and vendor.tgz -.PHONY: vendor-remove -vendor-remove: - $(foreach plugin_dir,$(PLUGIN_VENDOR), \ - $(RM) $(plugin_dir)/vendor; \ + @echo "Vendoring dependencies" + @$(GOCMD) mod vendor + @$(foreach plugin,$(PLUGINS), \ + $(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \ ) - $(RM) vendor vendor.tgz .PHONY: package package: @@ -244,16 +174,13 @@ else @if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; } endif -# build a release tarball .PHONY: release release: check_release build package -# build the windows installer .PHONY: windows_installer windows_installer: build @.\make_installer.ps1 -version $(BUILD_VERSION) -# build the chocolatey package .PHONY: chocolatey chocolatey: windows_installer @.\make_chocolatey.ps1 -version $(BUILD_VERSION) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b1564b375f3..c529ee2faa7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ stages: - task: GoTool@0 displayName: "Install Go 1.20" inputs: - version: '1.20.5' + version: '1.20.4' - pwsh: | choco install -y make @@ -38,7 +38,7 @@ stages: pwsh: true #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI script: | - make build BUILD_RE2_WASM=1 + make build - task: AzureKeyVault@2 inputs: azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)' diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 6abe3db5afc..25cb26515d7 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -15,6 +15,7 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -154,6 +155,7 @@ var alertTemplate = ` ` + func DisplayOneAlert(alert *models.Alert, withDetail bool) error { if csConfig.Cscli.Output == "human" { tmpl, err := template.New("alert").Parse(alertTemplate) @@ -209,11 +211,11 @@ func NewAlertsCmd() *cobra.Command { PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("parsing api url %s: %w", apiURL, err) + return errors.Wrapf(err, "parsing api url %s", apiURL) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -224,7 +226,7 @@ func NewAlertsCmd() *cobra.Command { }) if err != nil { - return fmt.Errorf("new api client: %w", err) + return errors.Wrap(err, "new api client") } return nil }, diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index af6e9c2e86f..e67d33ce458 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -6,11 +6,6 @@ import ( "net/url" "os" - "github.com/go-openapi/strfmt" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -19,6 +14,11 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" ) const CAPIBaseURL string = "https://api.crowdsec.net/" @@ -31,11 +31,8 @@ func NewCapiCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadAPIServer(); err != nil { - return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err) - } - if csConfig.DisableAPI { - return nil + if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { + return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath) @@ -136,7 +133,7 @@ func NewCapiStatusCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { var err error if csConfig.API.Server == nil { - log.Fatal("There is no configuration on 'api.server:'") + log.Fatalln("There is no configuration on 'api.server:'") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 717fc990b9b..30cf729fe1c 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -5,9 +5,11 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -32,17 +34,17 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) if _, err := os.Stat(parentDir); err != nil { - return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) + return errors.Wrapf(err, "while checking parent directory %s existence", parentDir) } if err = os.Mkdir(dirPath, 0o700); err != nil { - return fmt.Errorf("while creating %s: %w", dirPath, err) + return errors.Wrapf(err, "while creating %s", dirPath) } if csConfig.ConfigPaths.SimulationFilePath != "" { backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err) + if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { + return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation) } log.Infof("Saved simulation to %s", backupSimulation) @@ -54,14 +56,14 @@ func backupConfigToDirectory(dirPath string) error { */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) + if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } } acquisBackupDir := filepath.Join(dirPath, "acquis") if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err) + return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err) } if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { @@ -73,11 +75,11 @@ func backupConfigToDirectory(dirPath string) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) + if err = types.CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } log.Infof("Saved acquis %s to %s", acquisFile, targetFname) @@ -86,8 +88,8 @@ func backupConfigToDirectory(dirPath string) error { if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err) + if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err) } log.Infof("Saved default yaml to %s", backupMain) @@ -95,8 +97,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) + if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) } log.Infof("Saved online API credentials to %s", backupCAPICreds) @@ -104,8 +106,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) + if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) } log.Infof("Saved local API credentials to %s", backupLAPICreds) @@ -113,20 +115,21 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" { backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) + if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) } log.Infof("Saved profiles to %s", backupProfiles) } if err = BackupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config: %s", err) + return fmt.Errorf("failed to backup hub config : %s", err) } return nil } + func runConfigBackup(cmd *cobra.Command, args []string) error { if err := csConfig.LoadHub(); err != nil { return err @@ -144,6 +147,7 @@ func runConfigBackup(cmd *cobra.Command, args []string) error { return nil } + func NewConfigBackupCmd() *cobra.Command { cmdConfigBackup := &cobra.Command{ Use: `backup "directory"`, diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 55ab7aa9bad..79d36d428f3 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -7,11 +7,13 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -36,7 +38,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if _, err = os.Stat(backupMain); err == nil { if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" { - if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { + if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err) } } @@ -49,21 +51,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) if _, err = os.Stat(backupCAPICreds); err == nil { - if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { + if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err) } } backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) if _, err = os.Stat(backupLAPICreds); err == nil { - if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { + if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err) } } backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) if _, err = os.Stat(backupProfiles); err == nil { - if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { + if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err) } } @@ -104,7 +106,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) if _, err = os.Stat(backupSimulation); err == nil { - if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { + if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err) } } @@ -121,7 +123,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if _, err = os.Stat(backupAcquisition); err == nil { log.Debugf("restoring backup'ed %s", backupAcquisition) - if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { + if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } @@ -132,12 +134,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { for _, acquisFile := range acquisFiles { targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname) } log.Debugf("restoring %s to %s", acquisFile, targetFname) - if err = CopyFile(acquisFile, targetFname); err != nil { + if err = types.CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } } @@ -155,10 +157,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) } - if err = CopyFile(acquisFile, targetFname); err != nil { + if err = types.CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } @@ -173,6 +175,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { return nil } + func runConfigRestore(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -197,6 +200,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return nil } + func NewConfigRestoreCmd() *cobra.Command { cmdConfigRestore := &cobra.Command{ Use: `restore "directory"`, diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go deleted file mode 100644 index 4de6cd6e24a..00000000000 --- a/cmd/crowdsec-cli/copyfile.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "fmt" - "io" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - - -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return - } - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - if os.SameFile(sourceFileStat, destinationFileStat) { - return - } - } - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - return -} - diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index ce3d0e46e2b..f2f3efcf822 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -7,15 +7,19 @@ import ( "fmt" "net/url" "os" + "path/filepath" "strconv" "strings" "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/jszwec/csvutil" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -108,12 +112,12 @@ func NewDecisionsCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } password := strfmt.Password(csConfig.API.Client.Credentials.Password) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err) + return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -123,7 +127,7 @@ func NewDecisionsCmd() *cobra.Command { VersionPrefix: "v1", }) if err != nil { - return fmt.Errorf("creating api client: %w", err) + return errors.Wrap(err, "creating api client") } return nil }, @@ -165,11 +169,11 @@ cscli decisions list -t ban `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error /*take care of shorthand options*/ - if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { - return err + if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { + log.Fatalf("%s", err) } filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true @@ -185,7 +189,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Until = fmt.Sprintf("%d%s", days*24, "h") } @@ -198,7 +202,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) + log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Since = fmt.Sprintf("%d%s", days*24, "h") } @@ -234,15 +238,13 @@ cscli decisions list -t ban alerts, _, err := Client.Alerts.List(context.Background(), filter) if err != nil { - return fmt.Errorf("unable to retrieve decisions: %w", err) + log.Fatalf("Unable to list decisions : %v", err) } err = DecisionsToTable(alerts, printMachine) if err != nil { - return fmt.Errorf("unable to print decisions: %w", err) + log.Fatalf("unable to list decisions : %v", err) } - - return nil }, } cmdDecisionsList.Flags().SortFlags = false @@ -286,7 +288,7 @@ cscli decisions add --scope username --value foobar /*TBD : fix long and example*/ Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error alerts := models.AddAlertsRequest{} origin := types.CscliOrigin @@ -301,7 +303,7 @@ cscli decisions add --scope username --value foobar /*take care of shorthand options*/ if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil { - return err + log.Fatalf("%s", err) } if addIP != "" { @@ -312,7 +314,7 @@ cscli decisions add --scope username --value foobar addScope = types.Range } else if addValue == "" { printHelp(cmd) - return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)") + log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)") } if addReason == "" { @@ -355,11 +357,10 @@ cscli decisions add --scope username --value foobar _, _, err = Client.Alerts.Add(context.Background(), alerts) if err != nil { - return err + log.Fatal(err) } log.Info("Decision successfully added") - return nil }, } @@ -400,27 +401,25 @@ cscli decisions delete --id 42 cscli decisions delete --type captcha `, /*TBD : refaire le Long/Example*/ - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRun: func(cmd *cobra.Command, args []string) { if delDecisionAll { - return nil + return } if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && *delFilter.OriginEquals == "" && delDecisionId == "" { cmd.Usage() - return fmt.Errorf("at least one filter or --all must be specified") + log.Fatalln("At least one filter or --all must be specified") } - - return nil }, - RunE: func(cmd *cobra.Command, args []string) error { + Run: func(cmd *cobra.Command, args []string) { var err error var decisions *models.DeleteDecisionResponse /*take care of shorthand options*/ - if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { - return err + if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { + log.Fatalf("%s", err) } if *delFilter.ScopeEquals == "" { delFilter.ScopeEquals = nil @@ -450,19 +449,18 @@ cscli decisions delete --type captcha if delDecisionId == "" { decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter) if err != nil { - return fmt.Errorf("Unable to delete decisions: %v", err) + log.Fatalf("Unable to delete decisions : %v", err) } } else { if _, err = strconv.Atoi(delDecisionId); err != nil { - return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err) + log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err) } decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId) if err != nil { - return fmt.Errorf("Unable to delete decision: %v", err) + log.Fatalf("Unable to delete decision : %v", err) } } log.Infof("%s decision(s) deleted", decisions.NbDeleted) - return nil }, } @@ -480,3 +478,192 @@ cscli decisions delete --type captcha return cmdDecisionsDelete } + +func NewDecisionsImportCmd() *cobra.Command { + var ( + defaultDuration = "4h" + defaultScope = "ip" + defaultType = "ban" + defaultReason = "manual" + importDuration string + importScope string + importReason string + importType string + importFile string + batchSize int + ) + + var cmdDecisionImport = &cobra.Command{ + Use: "import [options]", + Short: "Import decisions from json or csv file", + Long: "expected format :\n" + + "csv : any of duration,origin,reason,scope,type,value, with a header line\n" + + `json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, + DisableAutoGenTag: true, + Example: `decisions.csv : +duration,scope,value +24h,ip,1.2.3.4 + +cscsli decisions import -i decisions.csv + +decisions.json : +[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] +`, + Run: func(cmd *cobra.Command, args []string) { + if importFile == "" { + log.Fatalf("Please provide a input file containing decisions with -i flag") + } + csvData, err := os.ReadFile(importFile) + if err != nil { + log.Fatalf("unable to open '%s': %s", importFile, err) + } + type decisionRaw struct { + Duration string `csv:"duration,omitempty" json:"duration,omitempty"` + Origin string `csv:"origin,omitempty" json:"origin,omitempty"` + Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` + Scope string `csv:"scope,omitempty" json:"scope,omitempty"` + Type string `csv:"type,omitempty" json:"type,omitempty"` + Value string `csv:"value" json:"value"` + } + var decisionsListRaw []decisionRaw + switch fileFormat := filepath.Ext(importFile); fileFormat { + case ".json": + if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil { + log.Fatalf("unable to unmarshall json: '%s'", err) + } + case ".csv": + if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil { + log.Fatalf("unable to unmarshall csv: '%s'", err) + } + default: + log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile) + } + + decisionsList := make([]*models.Decision, 0) + for i, decisionLine := range decisionsListRaw { + line := i + 2 + if decisionLine.Value == "" { + log.Fatalf("please provide a 'value' in your csv line %d", line) + } + /*deal with defaults and cli-override*/ + if decisionLine.Duration == "" { + decisionLine.Duration = defaultDuration + log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration) + } + if importDuration != "" { + decisionLine.Duration = importDuration + log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration) + } + decisionLine.Origin = types.CscliImportOrigin + + if decisionLine.Scenario == "" { + decisionLine.Scenario = defaultReason + log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario) + } + if importReason != "" { + decisionLine.Scenario = importReason + log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason) + } + if decisionLine.Type == "" { + decisionLine.Type = defaultType + log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type) + } + if importType != "" { + decisionLine.Type = importType + log.Debugf("'type' line %d, using supplied value: '%s'", line, importType) + } + if decisionLine.Scope == "" { + decisionLine.Scope = defaultScope + log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope) + } + if importScope != "" { + decisionLine.Scope = importScope + log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope) + } + decision := models.Decision{ + Value: ptr.Of(decisionLine.Value), + Duration: ptr.Of(decisionLine.Duration), + Origin: ptr.Of(decisionLine.Origin), + Scenario: ptr.Of(decisionLine.Scenario), + Type: ptr.Of(decisionLine.Type), + Scope: ptr.Of(decisionLine.Scope), + Simulated: new(bool), + } + decisionsList = append(decisionsList, &decision) + } + alerts := models.AddAlertsRequest{} + + if batchSize > 0 { + for i := 0; i < len(decisionsList); i += batchSize { + end := i + batchSize + if end > len(decisionsList) { + end = len(decisionsList) + } + decisionBatch := decisionsList[i:end] + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))), + + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(decisionBatch))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: decisionBatch, + } + alerts = append(alerts, &importAlert) + } + } else { + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))), + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(decisionsList))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: decisionsList, + } + alerts = append(alerts, &importAlert) + } + + if len(decisionsList) > 1000 { + log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList)) + } + + _, _, err = Client.Alerts.Add(context.Background(), alerts) + if err != nil { + log.Fatal(err) + } + log.Infof("%d decisions successfully imported", len(decisionsList)) + }, + } + + cmdDecisionImport.Flags().SortFlags = false + cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file") + cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)") + cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)") + cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)") + cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)") + cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions") + + return cmdDecisionImport +} diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go deleted file mode 100644 index 6a47a96b3ea..00000000000 --- a/cmd/crowdsec-cli/decisions_import.go +++ /dev/null @@ -1,272 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/jszwec/csvutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" - "github.com/crowdsecurity/go-cs-lib/pkg/slicetools" - - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" -) - -// decisionRaw is only used to unmarshall json/csv decisions -type decisionRaw struct { - Duration string `csv:"duration,omitempty" json:"duration,omitempty"` - Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` - Scope string `csv:"scope,omitempty" json:"scope,omitempty"` - Type string `csv:"type,omitempty" json:"type,omitempty"` - Value string `csv:"value" json:"value"` -} - -func parseDecisionList(content []byte, format string) ([]decisionRaw, error) { - ret := []decisionRaw{} - - switch format { - case "values": - log.Infof("Parsing values") - scanner := bufio.NewScanner(bytes.NewReader(content)) - for scanner.Scan() { - value := strings.TrimSpace(scanner.Text()) - ret = append(ret, decisionRaw{Value: value}) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("unable to parse values: '%s'", err) - } - case "json": - log.Infof("Parsing json") - if err := json.Unmarshal(content, &ret); err != nil { - return nil, err - } - case "csv": - log.Infof("Parsing csv") - if err := csvutil.Unmarshal(content, &ret); err != nil { - return nil, fmt.Errorf("unable to parse csv: '%s'", err) - } - default: - return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format) - } - - return ret, nil -} - - -func runDecisionsImport(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - input, err := flags.GetString("input") - if err != nil { - return err - } - - defaultDuration, err := flags.GetString("duration") - if err != nil { - return err - } - if defaultDuration == "" { - return fmt.Errorf("--duration cannot be empty") - } - - defaultScope, err := flags.GetString("scope") - if err != nil { - return err - } - if defaultScope == "" { - return fmt.Errorf("--scope cannot be empty") - } - - defaultReason, err := flags.GetString("reason") - if err != nil { - return err - } - if defaultReason == "" { - return fmt.Errorf("--reason cannot be empty") - } - - defaultType, err := flags.GetString("type") - if err != nil { - return err - } - if defaultType == "" { - return fmt.Errorf("--type cannot be empty") - } - - batchSize, err := flags.GetInt("batch") - if err != nil { - return err - } - - format, err := flags.GetString("format") - if err != nil { - return err - } - - var ( - content []byte - fin *os.File - ) - - // set format if the file has a json or csv extension - if format == "" { - if strings.HasSuffix(input, ".json") { - format = "json" - } else if strings.HasSuffix(input, ".csv") { - format = "csv" - } - } - - if format == "" { - return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag") - } - - if input == "-" { - fin = os.Stdin - input = "stdin" - } else { - fin, err = os.Open(input) - if err != nil { - return fmt.Errorf("unable to open %s: %s", input, err) - } - } - - content, err = io.ReadAll(fin) - if err != nil { - return fmt.Errorf("unable to read from %s: %s", input, err) - } - - decisionsListRaw, err := parseDecisionList(content, format) - if err != nil { - return err - } - - decisions := make([]*models.Decision, len(decisionsListRaw)) - for i, d := range decisionsListRaw { - if d.Value == "" { - return fmt.Errorf("item %d: missing 'value'", i) - } - - if d.Duration == "" { - d.Duration = defaultDuration - log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration) - } - - if d.Scenario == "" { - d.Scenario = defaultReason - log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason) - } - - if d.Type == "" { - d.Type = defaultType - log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType) - } - - if d.Scope == "" { - d.Scope = defaultScope - log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope) - } - - decisions[i] = &models.Decision{ - Value: ptr.Of(d.Value), - Duration: ptr.Of(d.Duration), - Origin: ptr.Of(types.CscliImportOrigin), - Scenario: ptr.Of(d.Scenario), - Type: ptr.Of(d.Type), - Scope: ptr.Of(d.Scope), - Simulated: ptr.Of(false), - } - } - - alerts := models.AddAlertsRequest{} - - for _, chunk := range slicetools.Chunks(decisions, batchSize) { - log.Debugf("Processing chunk of %d decisions", len(chunk)) - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))), - - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(chunk))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: chunk, - } - alerts = append(alerts, &importAlert) - } - - if len(decisions) > 1000 { - log.Infof("You are about to add %d decisions, this may take a while", len(decisions)) - } - - _, _, err = Client.Alerts.Add(context.Background(), alerts) - if err != nil { - return err - } - - log.Infof("Imported %d decisions", len(decisions)) - return nil -} - - -func NewDecisionsImportCmd() *cobra.Command { - var cmdDecisionsImport = &cobra.Command{ - Use: "import [options]", - Short: "Import decisions from a file or pipe", - Long: "expected format:\n" + - "csv : any of duration,reason,scope,type,value, with a header line\n" + - `json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, - DisableAutoGenTag: true, - Example: `decisions.csv: -duration,scope,value -24h,ip,1.2.3.4 - -$ cscli decisions import -i decisions.csv - -decisions.json: -[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] - -The file format is detected from the extension, but can be forced with the --format option -which is required when reading from standard input. - -Raw values, standard input: - -$ echo "1.2.3.4" | cscli decisions import -i - --format values -`, - RunE: runDecisionsImport, - } - - flags := cmdDecisionsImport.Flags() - flags.SortFlags = false - flags.StringP("input", "i", "", "Input file") - flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m") - flags.String("scope", types.Ip, "Decision scope: ip,range,username") - flags.StringP("reason", "R", "manual", "Decision reason: ") - flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle") - flags.Int("batch", 0, "Split import in batches of N decisions") - flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)") - - cmdDecisionsImport.MarkFlagRequired("input") - - return cmdDecisionsImport -} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index d03662c4896..e8a29d9cdbc 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -23,6 +24,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" + "github.com/crowdsecurity/crowdsec/pkg/types" ) var LAPIURLPrefix string = "v1" @@ -203,7 +205,7 @@ func NewLapiCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrap(err, "loading api client") } return nil }, @@ -438,7 +440,7 @@ cscli lapi context delete --value evt.Line.Src return cmdContext } -func detectStaticField(GrokStatics []parser.ExtraField) []string { +func detectStaticField(GrokStatics []types.ExtraField) []string { ret := make([]string, 0) for _, static := range GrokStatics { if static.Parsed != "" { diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 21594310230..25bd5acec40 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -12,6 +12,7 @@ import ( "time" "github.com/AlecAivazis/survey/v2" + "github.com/enescakir/emoji" "github.com/fatih/color" "github.com/go-openapi/strfmt" "github.com/google/uuid" @@ -84,21 +85,22 @@ func generateID(prefix string) (string, error) { return prefix + suffix, nil } -// getLastHeartbeat returns the last heartbeat timestamp of a machine -// and a boolean indicating if the machine is considered active or not. -func getLastHeartbeat(m *ent.Machine) (string, bool) { - if m.LastHeartbeat == nil { - return "-", false - } - - elapsed := time.Now().UTC().Sub(*m.LastHeartbeat) +func displayLastHeartBeat(m *ent.Machine, fancy bool) string { + var hbDisplay string - hb := elapsed.Truncate(time.Second).String() - if elapsed > 2*time.Minute { - return hb, false + if m.LastHeartbeat != nil { + lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat) + hbDisplay = lastHeartBeat.Truncate(time.Second).String() + if fancy && lastHeartBeat > 2*time.Minute { + hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String()) + } + } else { + hbDisplay = "-" + if fancy { + hbDisplay = emoji.Warning.String() + " -" + } } - - return hb, true + return hbDisplay } func getAgents(out io.Writer, dbClient *database.Client) error { @@ -128,10 +130,9 @@ func getAgents(out io.Writer, dbClient *database.Client) error { } else { validated = "false" } - hb, _ := getLastHeartbeat(m) - err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}) + err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)}) if err != nil { - return fmt.Errorf("failed to write raw output: %w", err) + return fmt.Errorf("failed to write raw output : %s", err) } } csvwriter.Flush() diff --git a/cmd/crowdsec-cli/machines_table.go b/cmd/crowdsec-cli/machines_table.go index e166fb785a6..cc15bb51bd3 100644 --- a/cmd/crowdsec-cli/machines_table.go +++ b/cmd/crowdsec-cli/machines_table.go @@ -24,11 +24,7 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) { validated = emoji.Prohibited.String() } - hb, active := getLastHeartbeat(m) - if !active { - hb = emoji.Warning.String() + " " + hb - } - t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb) + t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true)) } t.Render() diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index fe4c14f27c5..ebbefd6b722 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -15,6 +15,7 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/tomb.v2" @@ -27,12 +28,14 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csprofiles" ) + type NotificationsCfg struct { Config csplugin.PluginConfig `json:"plugin_config"` Profiles []*csconfig.ProfileCfg `json:"associated_profiles"` ids []uint } + func NewNotificationsCmd() *cobra.Command { var cmdNotifications = &cobra.Command{ Use: "notifications [action]", @@ -54,6 +57,7 @@ func NewNotificationsCmd() *cobra.Command { }, } + cmdNotifications.AddCommand(NewNotificationsListCmd()) cmdNotifications.AddCommand(NewNotificationsInspectCmd()) cmdNotifications.AddCommand(NewNotificationsReinjectCmd()) @@ -61,17 +65,18 @@ func NewNotificationsCmd() *cobra.Command { return cmdNotifications } + func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { pcfgs := map[string]csplugin.PluginConfig{} wf := func(path string, info fs.FileInfo, err error) error { if info == nil { - return fmt.Errorf("error while traversing directory %s: %w", path, err) + return errors.Wrapf(err, "error while traversing directory %s", path) } name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) { ts, err := csplugin.ParsePluginConfigFile(name) if err != nil { - return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err) + return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name) } for _, t := range ts { pcfgs[t.Name] = t @@ -81,14 +86,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { } if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil { - return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err) + return nil, errors.Wrap(err, "Loading notifification plugin configuration") } // A bit of a tricky stuf now: reconcile profiles and notification plugins ncfgs := map[string]NotificationsCfg{} profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return nil, fmt.Errorf("while extracting profiles from configuration: %w", err) + return nil, errors.Wrap(err, "Cannot extract profiles from configuration") } for profileID, profile := range profiles { loop: @@ -124,6 +129,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { return ncfgs, nil } + func NewNotificationsListCmd() *cobra.Command { var cmdNotificationsList = &cobra.Command{ Use: "list", @@ -135,7 +141,7 @@ func NewNotificationsListCmd() *cobra.Command { RunE: func(cmd *cobra.Command, arg []string) error { ncfgs, err := getNotificationsConfiguration() if err != nil { - return fmt.Errorf("can't build profiles configuration: %w", err) + return errors.Wrap(err, "Can't build profiles configuration") } if csConfig.Cscli.Output == "human" { @@ -143,14 +149,14 @@ func NewNotificationsListCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(ncfgs, "", " ") if err != nil { - return fmt.Errorf("failed to marshal notification configuration: %w", err) + return errors.New("failed to marshal notification configuration") } fmt.Printf("%s", string(x)) } else if csConfig.Cscli.Output == "raw" { csvwriter := csv.NewWriter(os.Stdout) err := csvwriter.Write([]string{"Name", "Type", "Profile name"}) if err != nil { - return fmt.Errorf("failed to write raw header: %w", err) + return errors.Wrap(err, "failed to write raw header") } for _, b := range ncfgs { profilesList := []string{} @@ -159,7 +165,7 @@ func NewNotificationsListCmd() *cobra.Command { } err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")}) if err != nil { - return fmt.Errorf("failed to write raw content: %w", err) + return errors.Wrap(err, "failed to write raw content") } } csvwriter.Flush() @@ -171,6 +177,7 @@ func NewNotificationsListCmd() *cobra.Command { return cmdNotificationsList } + func NewNotificationsInspectCmd() *cobra.Command { var cmdNotificationsInspect = &cobra.Command{ Use: "inspect", @@ -188,14 +195,14 @@ func NewNotificationsInspectCmd() *cobra.Command { pluginName := arg[0] if pluginName == "" { - return fmt.Errorf("please provide a plugin name to inspect") + errors.New("Please provide a plugin name to inspect") } ncfgs, err := getNotificationsConfiguration() if err != nil { - return fmt.Errorf("can't build profiles configuration: %w", err) + return errors.Wrap(err, "Can't build profiles configuration") } if cfg, ok = ncfgs[pluginName]; !ok { - return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName) + return errors.New("The provided plugin name doesn't exist or isn't active") } if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" { @@ -209,7 +216,7 @@ func NewNotificationsInspectCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(cfg, "", " ") if err != nil { - return fmt.Errorf("failed to marshal notification configuration: %w", err) + return errors.New("failed to marshal notification configuration") } fmt.Printf("%s", string(x)) } @@ -220,6 +227,7 @@ func NewNotificationsInspectCmd() *cobra.Command { return cmdNotificationsInspect } + func NewNotificationsReinjectCmd() *cobra.Command { var remediation bool var alertOverride string @@ -242,26 +250,26 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not ) if len(args) != 1 { printHelp(cmd) - return fmt.Errorf("wrong number of argument: there should be one argument") + return errors.New("Wrong number of argument: there should be one argument") } //first: get the alert id, err := strconv.Atoi(args[0]) if err != nil { - return fmt.Errorf("bad alert id %s", args[0]) + return errors.New(fmt.Sprintf("bad alert id %s", args[0])) } if err := csConfig.LoadAPIClient(); err != nil { - return fmt.Errorf("loading api client: %w", err) + return errors.Wrapf(err, "loading api client") } if csConfig.API.Client == nil { - return fmt.Errorf("missing configuration on 'api_client:'") + return errors.New("There is no configuration on 'api_client:'") } if csConfig.API.Client.Credentials == nil { - return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath) + return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return fmt.Errorf("error parsing the URL of the API: %w", err) + return errors.Wrapf(err, "error parsing the URL of the API") } client, err := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -271,16 +279,16 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not VersionPrefix: "v1", }) if err != nil { - return fmt.Errorf("error creating the client for the API: %w", err) + return errors.Wrapf(err, "error creating the client for the API") } alert, _, err := client.Alerts.GetByID(context.Background(), id) if err != nil { - return fmt.Errorf("can't find alert with id %s: %w", args[0], err) + return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0])) } if alertOverride != "" { if err = json.Unmarshal([]byte(alertOverride), alert); err != nil { - return fmt.Errorf("can't unmarshal data in the alert flag: %w", err) + return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag") } } if !remediation { @@ -290,7 +298,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not // second we start plugins err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths) if err != nil { - return fmt.Errorf("can't initialize plugins: %w", err) + return errors.Wrapf(err, "Can't initialize plugins") } pluginTomb.Go(func() error { @@ -302,13 +310,13 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return fmt.Errorf("cannot extract profiles from configuration: %w", err) + return errors.Wrap(err, "Cannot extract profiles from configuration") } for id, profile := range profiles { _, matched, err := profile.EvaluateProfile(alert) if err != nil { - return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err) + return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name) } if !matched { log.Infof("The profile %s didn't match", profile.Cfg.Name) @@ -336,7 +344,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not } // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(fmt.Errorf("terminating")) + pluginTomb.Kill(errors.New("terminating")) pluginTomb.Wait() return nil }, diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index de52dcb4876..a5b433228b6 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/fatih/color" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -32,7 +33,7 @@ cscli scenarios remove crowdsecurity/ssh-bf } if err := cwhub.SetHubBranch(); err != nil { - return fmt.Errorf("while setting hub branch: %w", err) + return errors.Wrap(err, "while setting hub branch") } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 66c1493a4b6..013abf4b260 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -26,6 +26,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -47,14 +48,6 @@ const ( SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" ) -// from https://github.com/acarl005/stripansi -var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") - -func stripAnsiString(str string) string { - // the byte version doesn't strip correctly - return reStripAnsi.ReplaceAllString(str, "") -} - func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") err := csConfig.LoadPrometheus() @@ -407,7 +400,7 @@ cscli support dump -f /tmp/crowdsec-support.zip log.Errorf("Could not add zip entry for %s: %s", filename, err) continue } - fw.Write([]byte(stripAnsiString(string(data)))) + fw.Write([]byte(types.StripAnsiString(string(data)))) } err = zipWriter.Close() diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index f77342cc941..1a99c2acfa5 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", tfile.Name()) sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { + if err = types.CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", file.Name()) sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = CopyFile(sourceFile, destinationFile); err != nil { + if err = types.CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -657,7 +657,7 @@ func BackupHub(dirPath string) error { } clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { + if err = types.CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) } clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index fd2e2ce088c..3ce249d4cb2 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "runtime" "time" @@ -21,7 +20,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { apiServer, err := apiserver.NewServer(cConfig.API.Server) if err != nil { - return nil, fmt.Errorf("unable to run local API: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } if hasPlugins(cConfig.API.Server.Profiles) { @@ -30,27 +29,23 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { if cConfig.PluginConfig == nil && runtime.GOOS != "windows" { return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration") } - if cConfig.ConfigPaths.NotificationDir == "" { return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined") } - if cConfig.ConfigPaths.PluginDir == "" { return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined") } - err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths) if err != nil { - return nil, fmt.Errorf("unable to run plugin broker: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } - log.Info("initiated plugin broker") apiServer.AttachPluginBroker(&pluginBroker) } err = apiServer.InitController() if err != nil { - return nil, fmt.Errorf("unable to run local API: %w", err) + return nil, errors.Wrap(err, "unable to run local API") } return apiServer, nil diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 68a7c6180da..8b4487e158b 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -3,12 +3,10 @@ package main import ( "fmt" "os" - "path/filepath" "sync" "time" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "path/filepath" "github.com/crowdsecurity/go-cs-lib/pkg/trace" @@ -18,28 +16,31 @@ import ( leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error // Populate cwhub package tools - if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { - return nil, fmt.Errorf("while loading hub index: %w", err) + if err := cwhub.GetHubIdx(cConfig.Hub); err != nil { + return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err) } // Start loading configs csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return nil, fmt.Errorf("while loading parsers: %w", err) + return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) } if err := LoadBuckets(cConfig); err != nil { - return nil, fmt.Errorf("while loading scenarios: %w", err) + return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err) } if err := LoadAcquisition(cConfig); err != nil { - return nil, fmt.Errorf("while loading acquisition config: %w", err) + return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err) } return csParsers, nil } @@ -117,7 +118,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { aggregated = true } if err := acquisition.GetMetrics(dataSources, aggregated); err != nil { - return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err) + return errors.Wrap(err, "while fetching prometheus metrics for datasources.") } } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 0de6f037844..767097f0e2e 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -51,15 +51,12 @@ var ( ) type Flags struct { - ConfigFile string - - LogLevelTrace bool - LogLevelDebug bool - LogLevelInfo bool - LogLevelWarn bool - LogLevelError bool - LogLevelFatal bool - + ConfigFile string + TraceLevel bool + DebugLevel bool + InfoLevel bool + WarnLevel bool + ErrorLevel bool PrintVersion bool SingleFileType string Labels map[string]string @@ -110,7 +107,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error { dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform) if err != nil { - return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err) + return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN) } } else { dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec) @@ -119,10 +116,6 @@ func LoadAcquisition(cConfig *csconfig.Config) error { } } - if len(dataSources) == 0 { - return fmt.Errorf("no datasource enabled") - } - return nil } @@ -147,14 +140,11 @@ func (l labelsMap) Set(label string) error { func (f *Flags) Parse() { flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file") - - flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)") - flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'") - flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'") - flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'") - flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'") - flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'") - + flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose") + flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr") + flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr") + flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr") + flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr") flag.BoolVar(&f.PrintVersion, "version", false, "display version") flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine") flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition") @@ -182,18 +172,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { // override from flags switch { - case f.LogLevelTrace: + case f.TraceLevel: ret = log.TraceLevel - case f.LogLevelDebug: + case f.DebugLevel: ret = log.DebugLevel - case f.LogLevelInfo: + case f.InfoLevel: ret = log.InfoLevel - case f.LogLevelWarn: + case f.WarnLevel: ret = log.WarnLevel - case f.LogLevelError: + case f.ErrorLevel: ret = log.ErrorLevel - case f.LogLevelFatal: - ret = log.FatalLevel default: } diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 67489f459ff..17cc99827de 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -7,10 +7,6 @@ import ( "sync" "time" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -20,6 +16,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) { @@ -51,11 +50,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error alertsToPush, err := dedupAlerts(alerts) if err != nil { - return fmt.Errorf("failed to transform alerts for api: %w", err) + return errors.Wrap(err, "failed to transform alerts for api") } _, _, err = client.Alerts.Add(ctx, alertsToPush) if err != nil { - return fmt.Errorf("failed sending alert to LAPI: %w", err) + return errors.Wrap(err, "failed sending alert to LAPI") } return nil } @@ -105,11 +104,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky Scenarios: scenarios, }) if err != nil { - return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err) + return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login) } if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { - return fmt.Errorf("unable to parse jwt expiration: %w", err) + return errors.Wrap(err, "unable to parse jwt expiration") } Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index 3f717e3975d..adb072376f9 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -12,7 +12,9 @@ import ( ) func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error { - count := 0 + var ( + count int + ) for { //bucket is now ready select { diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index d63a587ac16..c51d24147cd 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows/svc" @@ -21,7 +22,7 @@ func StartRunSvc() error { isRunninginService, err := svc.IsWindowsService() if err != nil { - return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err) + return errors.Wrap(err, "failed to determine if we are running in windows service mode") } if isRunninginService { return runService(svcName) @@ -30,22 +31,22 @@ func StartRunSvc() error { if flags.WinSvc == "Install" { err = installService(svcName, svcDescription) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Remove" { err = removeService(svcName) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Start" { err = startService(svcName) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "Stop" { err = controlService(svcName, svc.Stop, svc.Stopped) if err != nil { - return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) + return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) } } else if flags.WinSvc == "" { return WindowsRun() @@ -65,7 +66,7 @@ func WindowsRun() error { if err != nil { return err } - + // Configure logging log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 5d365b410d7..5e2e8b720a3 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -1,16 +1,16 @@ package main import ( - "fmt" "os" "os/signal" "syscall" "time" + "github.com/coreos/go-systemd/v22/daemon" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/csdaemon" "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } apiServer, err := initAPIServer(cConfig) if err != nil { - return nil, fmt.Errorf("unable to init api server: %w", err) + return nil, errors.Wrap(err, "unable to init api server") } apiReady := make(chan bool, 1) @@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return nil, fmt.Errorf("unable to init crowdsec: %w", err) + return nil, errors.Wrap(err, "unable to init crowdsec") } // restore bucket state @@ -180,13 +180,13 @@ func shutdownCrowdsec() error { func shutdown(sig os.Signal, cConfig *csconfig.Config) error { if !cConfig.DisableAgent { if err := shutdownCrowdsec(); err != nil { - return fmt.Errorf("failed to shut down crowdsec: %w", err) + return errors.Wrap(err, "failed to shut down crowdsec") } } if !cConfig.DisableAPI { if err := shutdownAPI(); err != nil { - return fmt.Errorf("failed to shut down api routines: %w", err) + return errors.Wrap(err, "failed to shut down api routines") } } @@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error { log.Warning("SIGHUP received, reloading") if err = shutdown(s, cConfig); err != nil { - exitChan <- fmt.Errorf("failed shutdown: %w", err) + exitChan <- errors.Wrap(err, "failed shutdown") break Loop } if newConfig, err = reloadHandler(s); err != nil { - exitChan <- fmt.Errorf("reload handler failure: %w", err) + exitChan <- errors.Wrap(err, "reload handler failure") break Loop } @@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error { case os.Interrupt, syscall.SIGTERM: log.Warning("SIGTERM received, shutting down") if err = shutdown(s, cConfig); err != nil { - exitChan <- fmt.Errorf("failed shutdown: %w", err) + exitChan <- errors.Wrap(err, "failed shutdown") break Loop } @@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil { dbClient, err := database.NewClient(cConfig.API.Server.DbConfig) if err != nil { - return fmt.Errorf("failed to get database client: %w", err) + return errors.Wrap(err, "failed to get database client") } err = exprhelpers.Init(dbClient) if err != nil { - return fmt.Errorf("failed to init expr helpers: %w", err) + return errors.Wrap(err, "failed to init expr helpers") } } else { err := exprhelpers.Init(nil) if err != nil { - return fmt.Errorf("failed to init expr helpers: %w", err) + return errors.Wrap(err, "failed to init expr helpers") } log.Warningln("Exprhelpers loaded without database client.") @@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled { log.Infof("Crowdsec CTI helper enabled") if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil { - return fmt.Errorf("failed to init crowdsec cti: %w", err) + return errors.Wrap(err, "failed to init crowdsec cti") } } @@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e apiServer, err := initAPIServer(cConfig) if err != nil { - return fmt.Errorf("api server init: %w", err) + return errors.Wrap(err, "api server init") } if !flags.TestMode { @@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return fmt.Errorf("crowdsec init: %w", err) + return errors.Wrap(err, "crowdsec init") } // if it's just linting, we're done @@ -350,7 +350,10 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if cConfig.Common != nil && cConfig.Common.Daemonize { - csdaemon.NotifySystemd(log.StandardLogger()) + sent, err := daemon.SdNotify(false, daemon.SdNotifyReady) + if !sent || err != nil { + log.Errorf("Failed to notify(sent: %v): %v", sent, err) + } // wait for signals return HandleSignals(cConfig) } diff --git a/cmd/crowdsec/win_service.go b/cmd/crowdsec/win_service.go index ab9ecc8151f..d0e80c58a53 100644 --- a/cmd/crowdsec/win_service.go +++ b/cmd/crowdsec/win_service.go @@ -8,10 +8,10 @@ package main import ( - "fmt" "syscall" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" @@ -106,7 +106,7 @@ func runService(name string) error { winsvc := crowdsec_winservice{config: cConfig} if err := svc.Run(name, &winsvc); err != nil { - return fmt.Errorf("%s service failed: %w", name, err) + return errors.Wrapf(err, "%s service failed", name) } log.Infof("%s service stopped", name) diff --git a/debian/control b/debian/control index 4673284e7b4..d06d3888484 100644 --- a/debian/control +++ b/debian/control @@ -1,8 +1,6 @@ Source: crowdsec Maintainer: Crowdsec Team -Build-Depends: debhelper, bash -Section: admin -Priority: optional +Build-Depends: debhelper, bash, git Package: crowdsec Architecture: any diff --git a/debian/rules b/debian/rules index e6202a6f774..6683e5443f2 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f -export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ') +export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on @@ -11,10 +11,12 @@ override_dh_auto_clean: override_dh_auto_test: override_dh_auto_build: override_dh_auto_install: - - # just use the prebuilt binaries, otherwise: - # make build BUILD_RE_WASM=0 BUILD_STATIC=1 - +# mkdir /tmp/go +# echo $(go version) +# echo $($GOCMD version) +# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd .. +# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd .. + make build mkdir -p debian/crowdsec/usr/bin mkdir -p debian/crowdsec/etc/crowdsec mkdir -p debian/crowdsec/usr/share/crowdsec diff --git a/debian/templates b/debian/templates index c6998eb8535..c07ef8446d4 100644 --- a/debian/templates +++ b/debian/templates @@ -17,7 +17,7 @@ Description: Address of the local API server Template: crowdsec/capi Type: boolean Default: true -Description: Do you want to use the centralized remote API server ? +Description: Do you want to the centralized remote API server ? To share information with other crowdsec you can register to the centralized remote API server. . If you don't know what to do, answer yes. diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 21b42dcb0b4..8ec44910311 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -56,7 +56,7 @@ conf_get() { if [ $# -ge 2 ]; then yq e "$1" "$2" else - cscli config show-yaml | yq e "$1" + yq e "$1" "$CONFIG_FILE" fi } diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index b890bebb9c6..81567954bc7 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -6,8 +6,11 @@ from http import HTTPStatus import json +import os +import pwd import pytest +import yaml pytestmark = pytest.mark.docker @@ -82,7 +85,12 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): 'COLLECTIONS': f'{coll}' } - with crowdsec(flavor=flavor, environment=env) as cs: + hub = tmp_path_factory.mktemp("hub") + volumes = { + hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'} + } + + with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run('cscli collections list -o json') assert res.exit_code == 0 @@ -94,13 +102,25 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): f'*Enabled collections : {coll}*', ]) - scenario = 'crowdsecurity/http-crawl-non_statics' - - # the description won't be read back, it's from the index - yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml" - res = cs.cont.exec_run(yq_command) + # change file permissions to allow edit + current_uid = pwd.getpwuid(os.getuid()).pw_uid + res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub') assert res.exit_code == 0 + scenario = 'crowdsecurity/http-crawl-non_statics' + scenario_file = hub / f'scenarios/{scenario}.yaml' + + with open(scenario_file) as f: + yml = yaml.safe_load(f) + + yml['description'] += ' (tainted)' + # won't be able to read it back because description is taken from the index + + with open(scenario_file, 'w') as f: + yaml.dump(yml, f) + + with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: + cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json') assert res.exit_code == 0 j = json.loads(res.output) diff --git a/go.mod b/go.mod index f547709e809..d64e1c0754b 100644 --- a/go.mod +++ b/go.mod @@ -5,22 +5,14 @@ go 1.20 require ( entgo.io/ent v0.11.3 github.com/AlecAivazis/survey/v2 v2.2.7 - github.com/Masterminds/semver/v3 v3.1.1 - github.com/Masterminds/sprig/v3 v3.2.2 + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alexliesenfeld/health v0.5.1 github.com/antonmedv/expr v1.12.5 github.com/appleboy/gin-jwt/v2 v2.8.0 - github.com/aquasecurity/table v1.8.0 - github.com/aws/aws-lambda-go v1.38.0 github.com/aws/aws-sdk-go v1.42.25 - github.com/beevik/etree v1.1.0 - github.com/blackfireio/osinfo v1.0.3 - github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 github.com/c-robinson/iplib v1.0.3 - github.com/cespare/xxhash/v2 v2.1.2 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.2 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 @@ -28,7 +20,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 - github.com/fatih/color v1.15.0 + github.com/fatih/color v1.13.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.17.0 @@ -37,50 +29,65 @@ require ( github.com/go-openapi/swag v0.19.14 github.com/go-openapi/validate v0.20.0 github.com/go-sql-driver/mysql v1.6.0 - github.com/goccy/go-yaml v1.9.7 - github.com/gofrs/uuid v4.0.0+incompatible - github.com/golang-jwt/jwt/v4 v4.2.0 github.com/google/go-querystring v1.0.0 github.com/google/uuid v1.3.0 - github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/hashicorp/go-hclog v1.0.0 github.com/hashicorp/go-plugin v1.4.2 github.com/hashicorp/go-version v1.2.1 - github.com/ivanpirog/coloredcobra v1.0.1 github.com/jackc/pgx/v4 v4.14.1 github.com/jarcoal/httpmock v1.1.0 github.com/jszwec/csvutil v1.5.1 - github.com/lithammer/dedent v1.1.0 - github.com/mattn/go-isatty v0.0.17 github.com/mattn/go-sqlite3 v1.14.16 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nxadm/tail v1.4.8 github.com/oschwald/geoip2-golang v1.4.0 github.com/oschwald/maxminddb-golang v1.8.0 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 - github.com/segmentio/kafka-go v0.4.34 - github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 - github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - github.com/wasilibs/go-re2 v0.2.1 golang.org/x/crypto v0.1.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/mod v0.11.0 - golang.org/x/sys v0.9.0 + golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 + gotest.tools/v3 v3.0.3 +) + +require ( + github.com/Masterminds/semver v1.5.0 + github.com/Masterminds/sprig/v3 v3.2.2 + github.com/aquasecurity/table v1.8.0 + github.com/aws/aws-lambda-go v1.38.0 + github.com/beevik/etree v1.1.0 + github.com/blackfireio/osinfo v1.0.3 + github.com/bluele/gcache v0.0.2 + github.com/cespare/xxhash/v2 v2.1.2 + github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 + github.com/coreos/go-systemd/v22 v22.5.0 + github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd + github.com/goccy/go-yaml v1.9.7 + github.com/gofrs/uuid v4.0.0+incompatible + github.com/golang-jwt/jwt/v4 v4.2.0 + github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b + github.com/ivanpirog/coloredcobra v1.0.1 + github.com/lithammer/dedent v1.1.0 + github.com/mattn/go-isatty v0.0.14 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.22.12 + github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c + github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 + github.com/wasilibs/go-re2 v0.2.1 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + golang.org/x/sys v0.7.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -88,7 +95,7 @@ require ( require ( ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/agext/levenshtein v1.2.1 // indirect @@ -96,7 +103,7 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -140,7 +147,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.14.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect @@ -155,6 +162,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -163,24 +171,24 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect - github.com/tidwall/gjson v1.13.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect - github.com/vmihailenco/msgpack v4.0 .4+incompatible // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect - go.mongodb.org/mongo-driver v1.9.4 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.6.0 // indirect + go.mongodb.org/mongo-driver v1.9.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/term v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -190,8 +198,11 @@ require ( k8s.io/apimachinery v0.25.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + rsc.io/binaryregexp v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 + +replace github.com/corazawaf/coraza/v3 => ./coraza diff --git a/go.sum b/go.sum index 41bff5d4779..8d45067581f 100644 --- a/go.sum +++ b/go.sum @@ -55,12 +55,14 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= @@ -149,6 +151,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= +github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -170,8 +174,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps= -github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -209,12 +213,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -657,18 +661,16 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -681,6 +683,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -755,6 +758,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -831,12 +836,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= +github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -887,6 +888,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -895,13 +897,14 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= -github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -940,8 +943,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -960,8 +963,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc= -go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck= +go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1028,8 +1031,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1052,8 +1055,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1104,8 +1107,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1125,8 +1128,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1198,23 +1201,23 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1224,8 +1227,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1294,7 +1297,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1465,6 +1467,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/mk/__gmsl b/mk/__gmsl deleted file mode 100644 index 6cd4a3090bc..00000000000 --- a/mk/__gmsl +++ /dev/null @@ -1,969 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2022 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# This is the GNU Make Standard Library version number as a list with -# three items: major, minor, revision - -gmsl_version := 1 2 0 - -__gmsl_name := GNU Make Standard Library - -# Used to output warnings and error from the library, it's possible to -# disable any warnings or errors by overriding these definitions -# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS - -ifdef GMSL_NO_WARNINGS -__gmsl_warning := -else -__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1)) -endif - -ifdef GMSL_NO_ERRORS -__gmsl_error := -else - __gmsl_error = $(if $1,$(error $(__gmsl_name): $1)) -endif - -# If GMSL_TRACE is enabled then calls to the library functions are -# traced to stdout using warning messages with their arguments - -ifdef GMSL_TRACE -__gmsl_tr1 = $(warning $0('$1')) -__gmsl_tr2 = $(warning $0('$1','$2')) -__gmsl_tr3 = $(warning $0('$1','$2','$3')) -else -__gmsl_tr1 := -__gmsl_tr2 := -__gmsl_tr3 := -endif - -# See if spaces are valid in variable names (this was the case until -# GNU Make 3.82) -ifeq ($(MAKE_VERSION),3.82) -__gmsl_spaced_vars := $(false) -else -__gmsl_spaced_vars := $(true) -endif - -# Figure out whether we have $(eval) or not (GNU Make 3.80 and above) -# if we do not then output a warning message, if we do then some -# functions will be enabled. - -__gmsl_have_eval := $(false) -__gmsl_ignore := $(eval __gmsl_have_eval := $(true)) - -# If this is being run with Electric Cloud's emake then warn that -# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0, -# 5.1, 5.2 and 5.3 - -ifdef ECLOUD_BUILD_ID -__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION))) -__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION))) -ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","") -$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support) -__gmsl_have_eval := $(false) -endif -endif - -# See if we have $(lastword) (GNU Make 3.81 and above) - -__gmsl_have_lastword := $(lastword $(false) $(true)) - -# See if we have native or and and (GNU Make 3.81 and above) - -__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/ -__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/ -__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false)) -__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false)) - -ifneq ($(__gmsl_have_eval),$(true)) -$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled) -endif - -__gmsl_dollar := $$ -__gmsl_hash := \# - -# ---------------------------------------------------------------------------- -# ---------------------------------------------------------------------------- -# Function: gmsl_compatible -# Arguments: List containing the desired library version number (maj min rev) -# Returns: $(true) if this version of the library is compatible -# with the requested version number, otherwise $(false) -# ---------------------------------------------------------------------------- -gmsl_compatible = $(strip \ - $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(true), \ - $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(true), \ - $(call lte,$(word 3,$1),$(word 3,$(gmsl_version)))))))) - -# ########################################################################### -# LOGICAL OPERATORS -# ########################################################################### - -# not is defined in gmsl - -# ---------------------------------------------------------------------------- -# Function: and -# Arguments: Two boolean values -# Returns: Returns $(true) if both of the booleans are true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_and),$(true)) -and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: or -# Arguments: Two boolean values -# Returns: Returns $(true) if either of the booleans is true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_or),$(true)) -or = $(__gmsl_tr2)$(if $1$2,$(true),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: xor -# Arguments: Two boolean values -# Returns: Returns $(true) if exactly one of the booleans is true -# ---------------------------------------------------------------------------- -xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false))) - -# ---------------------------------------------------------------------------- -# Function: nand -# Arguments: Two boolean values -# Returns: Returns value of 'not and' -# ---------------------------------------------------------------------------- -nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true)) - -# ---------------------------------------------------------------------------- -# Function: nor -# Arguments: Two boolean values -# Returns: Returns value of 'not or' -# ---------------------------------------------------------------------------- -nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: xnor -# Arguments: Two boolean values -# Returns: Returns value of 'not xor' -# ---------------------------------------------------------------------------- -xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true))) - -# ########################################################################### -# LIST MANIPULATION FUNCTIONS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: first (same as LISP's car, or head) -# Arguments: 1: A list -# Returns: Returns the first element of a list -# ---------------------------------------------------------------------------- -first = $(__gmsl_tr1)$(firstword $1) - -# ---------------------------------------------------------------------------- -# Function: last -# Arguments: 1: A list -# Returns: Returns the last element of a list -# ---------------------------------------------------------------------------- -ifeq ($(__gmsl_have_lastword),$(true)) -last = $(__gmsl_tr1)$(lastword $1) -else -last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1)) -endif - -# ---------------------------------------------------------------------------- -# Function: rest (same as LISP's cdr, or tail) -# Arguments: 1: A list -# Returns: Returns the list with the first element removed -# ---------------------------------------------------------------------------- -rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1) - -# ---------------------------------------------------------------------------- -# Function: chop -# Arguments: 1: A list -# Returns: Returns the list with the last element removed -# ---------------------------------------------------------------------------- -chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1) - -# ---------------------------------------------------------------------------- -# Function: map -# Arguments: 1: Name of function to $(call) for each element of list -# 2: List to iterate over calling the function in 1 -# Returns: The list after calling the function on each element -# ---------------------------------------------------------------------------- -map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a))) - -# ---------------------------------------------------------------------------- -# Function: pairmap -# Arguments: 1: Name of function to $(call) for each pair of elements -# 2: List to iterate over calling the function in 1 -# 3: Second list to iterate over calling the function in 1 -# Returns: The list after calling the function on each pair of elements -# ---------------------------------------------------------------------------- -pairmap = $(strip $(__gmsl_tr3)\ - $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \ - $(call pairmap,$1,$(call rest,$2),$(call rest,$3)))) - -# ---------------------------------------------------------------------------- -# Function: leq -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are identical -# ---------------------------------------------------------------------------- -leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \ - $(call __gmsl_list_equal,$1,$2),$(false))) - -__gmsl_list_equal = $(if $(strip $1), \ - $(if $(call seq,$(call first,$1),$(call first,$2)), \ - $(call __gmsl_list_equal, \ - $(call rest,$1), \ - $(call rest,$2)), \ - $(false)), \ - $(true)) - -# ---------------------------------------------------------------------------- -# Function: lne -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are different -# ---------------------------------------------------------------------------- -lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: reverse -# Arguments: 1: A list to reverse -# Returns: The list with its elements in reverse order -# ---------------------------------------------------------------------------- -reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \ - $(call first,$1))) - -# ---------------------------------------------------------------------------- -# Function: uniq -# Arguments: 1: A list from which to remove repeated elements -# Returns: The list with duplicate elements removed without reordering -# ---------------------------------------------------------------------------- -uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \ - $(call uniq,$(filter-out $(firstword $1),$1)))) - -# ---------------------------------------------------------------------------- -# Function: length -# Arguments: 1: A list -# Returns: The number of elements in the list -# ---------------------------------------------------------------------------- -length = $(__gmsl_tr1)$(words $1) - -# ########################################################################### -# STRING MANIPULATION FUNCTIONS -# ########################################################################### - -# Helper function that translates any GNU Make 'true' value (i.e. a -# non-empty string) to our $(true) - -__gmsl_make_bool = $(if $(strip $1),$(true),$(false)) - -# ---------------------------------------------------------------------------- -# Function: seq -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are identical -# ---------------------------------------------------------------------------- -seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: sne -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are not the same -# ---------------------------------------------------------------------------- -sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: split -# Arguments: 1: The character to split on -# 2: A string to split -# Returns: Splits a string into a list separated by spaces at the split -# character in the first argument -# ---------------------------------------------------------------------------- -split = $(__gmsl_tr2)$(strip $(subst $1, ,$2)) - -# ---------------------------------------------------------------------------- -# Function: merge -# Arguments: 1: The character to put between fields -# 2: A list to merge into a string -# Returns: Merges a list into a single string, list elements are separated -# by the character in the first argument -# ---------------------------------------------------------------------------- -merge = $(__gmsl_tr2)$(strip $(if $2, \ - $(if $(call seq,1,$(words $2)), \ - $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2))))) - -ifdef __gmsl_have_eval -# ---------------------------------------------------------------------------- -# Function: tr -# Arguments: 1: The list of characters to translate from -# 2: The list of characters to translate to -# 3: The text to translate -# Returns: Returns the text after translating characters -# ---------------------------------------------------------------------------- -tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \ - $(eval __gmsl_t := $3) \ - $(foreach c, \ - $(join $(addsuffix :,$1),$2), \ - $(eval __gmsl_t := \ - $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \ - $(__gmsl_t))))$(__gmsl_t)) - -# Common character classes for use with the tr function. Each of -# these is actually a variable declaration and must be wrapped with -# $() or ${} to be used. - -[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z # -[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z # -[0-9] := 0 1 2 3 4 5 6 7 8 9 # -[A-F] := A B C D E F # - -# ---------------------------------------------------------------------------- -# Function: uc -# Arguments: 1: Text to upper case -# Returns: Returns the text in upper case -# ---------------------------------------------------------------------------- -uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1) - -# ---------------------------------------------------------------------------- -# Function: lc -# Arguments: 1: Text to lower case -# Returns: Returns the text in lower case -# ---------------------------------------------------------------------------- -lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1) - -# ---------------------------------------------------------------------------- -# Function: strlen -# Arguments: 1: A string -# Returns: Returns the length of the string -# ---------------------------------------------------------------------------- - -# This results in __gmsl_tab containing a tab - -__gmsl_tab := # - -__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z -__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z -__gmsl_characters += 0 1 2 3 4 5 6 7 8 9 -__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = + -__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? | - -# This results in __gmsl_space containing just a space - -__gmsl_empty := -__gmsl_space := $(__gmsl_empty) $(__gmsl_empty) - -strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp))) - -# This results in __gmsl_newline containing just a newline - -define __gmsl_newline - - -endef - -# ---------------------------------------------------------------------------- -# Function: substr -# Arguments: 1: A string -# 2: Start position (first character is 1) -# 3: End position (inclusive) -# Returns: A substring. -# Note: The string in $1 must not contain a § -# ---------------------------------------------------------------------------- - -substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp)))) - -endif # __gmsl_have_eval - -# ########################################################################### -# SET MANIPULATION FUNCTIONS -# ########################################################################### - -# Sets are represented by sorted, deduplicated lists. To create a set -# from a list use set_create, or start with the empty_set and -# set_insert individual elements - -# This is the empty set -empty_set := - -# ---------------------------------------------------------------------------- -# Function: set_create -# Arguments: 1: A list of set elements -# Returns: Returns the newly created set -# ---------------------------------------------------------------------------- -set_create = $(__gmsl_tr1)$(sort $1) - -# ---------------------------------------------------------------------------- -# Function: set_insert -# Arguments: 1: A single element to add to a set -# 2: A set -# Returns: Returns the set with the element added -# ---------------------------------------------------------------------------- -set_insert = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_remove -# Arguments: 1: A single element to remove from a set -# 2: A set -# Returns: Returns the set with the element removed -# ---------------------------------------------------------------------------- -set_remove = $(__gmsl_tr2)$(filter-out $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_member, set_is_not_member -# Arguments: 1: A single element -# 2: A set -# Returns: (set_is_member) Returns $(true) if the element is in the set -# (set_is_not_member) Returns $(false) if the element is in the set -# ---------------------------------------------------------------------------- -set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false)) -set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: set_union -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the union of the two sets -# ---------------------------------------------------------------------------- -set_union = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_intersection -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the intersection of the two sets -# ---------------------------------------------------------------------------- -set_intersection = $(__gmsl_tr2)$(filter $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_subset -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the first set is a subset of the second -# ---------------------------------------------------------------------------- -set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1) - -# ---------------------------------------------------------------------------- -# Function: set_equal -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the two sets are identical -# ---------------------------------------------------------------------------- -set_equal = $(__gmsl_tr2)$(call seq,$1,$2) - -# ########################################################################### -# ARITHMETIC LIBRARY -# ########################################################################### - -# Integers a represented by lists with the equivalent number of x's. -# For example the number 4 is x x x x. - -# ---------------------------------------------------------------------------- -# Function: int_decode -# Arguments: 1: A number of x's representation -# Returns: Returns the integer for human consumption that is represented -# by the string of x's -# ---------------------------------------------------------------------------- -int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0) - -# ---------------------------------------------------------------------------- -# Function: int_encode -# Arguments: 1: A number in human-readable integer form -# Returns: Returns the integer encoded as a string of x's -# ---------------------------------------------------------------------------- -__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x)))) -__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0) -int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1)) - -# The arithmetic library functions come in two forms: one form of each -# function takes integers as arguments and the other form takes the -# encoded form (x's created by a call to int_encode). For example, -# there are two plus functions: -# -# plus Called with integer arguments and returns an integer -# int_plus Called with encoded arguments and returns an encoded result -# -# plus will be slower than int_plus because its arguments and result -# have to be translated between the x's format and integers. If doing -# a complex calculation use the int_* forms with a single encoding of -# inputs and single decoding of the output. For simple calculations -# the direct forms can be used. - -# Helper function used to wrap an int_* function into a function that -# takes a pair of integers, perhaps a function and returns an integer -# result -__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3))) -__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2))) -__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3)) - -# ---------------------------------------------------------------------------- -# Function: int_plus -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the sum of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_plus = $(strip $(__gmsl_tr2)$1 $2) - -# ---------------------------------------------------------------------------- -# Function: plus (wrapped version of int_plus) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the sum of the two integers -# ---------------------------------------------------------------------------- -plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_subtract -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the difference of the two numbers in x's representation, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \ - $(filter-out xx,$(join $1,$2)), \ - $(call __gmsl_warning,Subtraction underflow))) - -# ---------------------------------------------------------------------------- -# Function: subtract (wrapped version of int_subtract) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the difference of the two integers, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_multiply -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the product of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: multiply (wrapped version of int_multiply) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the product of the two integers -# ---------------------------------------------------------------------------- -multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_divide -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the result of integer division of argument 1 divided -# by argument 2 in x's representation -# ---------------------------------------------------------------------------- -int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ - $(subst M,x,$(filter-out x,$(subst $2,M,$1))), \ - $(call __gmsl_error,Division by zero)))) - -# ---------------------------------------------------------------------------- -# Function: divide (wrapped version of int_divide) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the integer division of the first argument by the second -# ---------------------------------------------------------------------------- -divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_modulo -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the remainder of integer division of argument 1 divided -# by argument 2 in x's representation -# ---------------------------------------------------------------------------- -int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ - $(filter-out M,$(subst $2,M,$1)), \ - $(call __gmsl_error,Division by zero)))) - -# ---------------------------------------------------------------------------- -# Function: modulo (wrapped version of int_modulo) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the remainder of integer division of the first argument -# by the second -# ---------------------------------------------------------------------------- -modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_max, int_min -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the maximum or minimum of its arguments in x's -# representation -# ---------------------------------------------------------------------------- -int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2)) -int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2))) - -# ---------------------------------------------------------------------------- -# Function: max, min -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the maximum or minimum of its integer arguments -# ---------------------------------------------------------------------------- -max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2) -min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne -# Arguments: Two x's representation numbers to be compared -# Returns: $(true) or $(false) -# -# int_gt First argument greater than second argument -# int_gte First argument greater than or equal to second argument -# int_lt First argument less than second argument -# int_lte First argument less than or equal to second argument -# int_eq First argument is numerically equal to the second argument -# int_ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $2), \ - $(words $(call int_max,$1,$2)))) -int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_gt,$1,$2)$(call int_eq,$1,$2)) -int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1), \ - $(words $(call int_max,$1,$2)))) -int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_lt,$1,$2)$(call int_eq,$1,$2)) -int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter $(words $1),$(words $2))) -int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1),$(words $2))) - -# ---------------------------------------------------------------------------- -# Function: gt, gte, lt, lte, eq, ne -# Arguments: Two integers to be compared -# Returns: $(true) or $(false) -# -# gt First argument greater than second argument -# gte First argument greater than or equal to second argument -# lt First argument less than second argument -# lte First argument less than or equal to second argument -# eq First argument is numerically equal to the second argument -# ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2) -gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2) -lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2) -lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2) -eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2) -ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2) - -# increment adds 1 to its argument, decrement subtracts 1. Note that -# decrement does not range check and hence will not underflow, but -# will incorrectly say that 0 - 1 = 0 - -# ---------------------------------------------------------------------------- -# Function: int_inc -# Arguments: 1: A number in x's representation -# Returns: The number incremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_inc = $(strip $(__gmsl_tr1)$1 x) - -# ---------------------------------------------------------------------------- -# Function: inc -# Arguments: 1: An integer -# Returns: The argument incremented by 1 -# ---------------------------------------------------------------------------- -inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1) - -# ---------------------------------------------------------------------------- -# Function: int_dec -# Arguments: 1: A number in x's representation -# Returns: The number decremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_dec = $(__gmsl_tr1)$(strip \ - $(if $(call sne,0,$(words $1)), \ - $(wordlist 2,$(words $1),$1))) - -# ---------------------------------------------------------------------------- -# Function: dec -# Arguments: 1: An integer -# Returns: The argument decremented by 1 -# ---------------------------------------------------------------------------- -dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1) - -# double doubles its argument, and halve halves it - -# ---------------------------------------------------------------------------- -# Function: int_double -# Arguments: 1: A number in x's representation -# Returns: The number doubled (i.e. * 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_double = $(strip $(__gmsl_tr1)$1 $1) - -# ---------------------------------------------------------------------------- -# Function: double -# Arguments: 1: An integer -# Returns: The integer times 2 -# ---------------------------------------------------------------------------- -double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1) - -# ---------------------------------------------------------------------------- -# Function: int_halve -# Arguments: 1: A number in x's representation -# Returns: The number halved (i.e. / 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \ - $(join $1,$(foreach a,$1,y x))))) - -# ---------------------------------------------------------------------------- -# Function: halve -# Arguments: 1: An integer -# Returns: The integer divided by 2 -# ---------------------------------------------------------------------------- -halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1) - -# ---------------------------------------------------------------------------- -# Function: sequence -# Arguments: 1: An integer -# 2: An integer -# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or -# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1] -# ---------------------------------------------------------------------------- -sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \ - $(call __gmsl_sequence_up,$1,$2), \ - $(call __gmsl_sequence_dn,$2,$1))) - -__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2)) -__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2))) - -# ---------------------------------------------------------------------------- -# Function: dec2hex, dec2bin, dec2oct -# Arguments: 1: An integer -# Returns: The decimal argument converted to hexadecimal, binary or -# octal -# ---------------------------------------------------------------------------- - -__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1)))))) - -dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16)) -dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2)) -dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8)) - -__gmsl_base_divide = $(subst $2,X ,$1) -__gmsl_q = $(strip $(filter X,$1)) -__gmsl_r = $(words $(filter x,$1)) - -__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2) -__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2) - -ifdef __gmsl_have_eval -# ########################################################################### -# ASSOCIATIVE ARRAYS -# ########################################################################### - -# Magic string that is very unlikely to appear in a key or value - -__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6 - -# ---------------------------------------------------------------------------- -# Function: set -# Arguments: 1: Name of associative array -# 2: The key value to associate -# 3: The value associated with the key -# Returns: Nothing -# ---------------------------------------------------------------------------- -set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3) - -# Only used internally by memoize function - -__gmsl_set = $(call set,$1,$2,$3)$3 - -# ---------------------------------------------------------------------------- -# Function: get -# Arguments: 1: Name of associative array -# 2: The key to retrieve -# Returns: The value stored in the array for that key -# ---------------------------------------------------------------------------- -get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1)) - -# ---------------------------------------------------------------------------- -# Function: keys -# Arguments: 1: Name of associative array -# Returns: Returns a list of all defined keys in the array -# ---------------------------------------------------------------------------- -keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \ - $(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES)))) - -# ---------------------------------------------------------------------------- -# Function: defined -# Arguments: 1: Name of associative array -# 2: The key to test -# Returns: Returns true if the key is defined (i.e. not empty) -# ---------------------------------------------------------------------------- -defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),) - -endif # __gmsl_have_eval - -ifdef __gmsl_have_eval -# ########################################################################### -# NAMED STACKS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: push -# Arguments: 1: Name of stack -# 2: Value to push onto the top of the stack (must not contain -# a space) -# Returns: None -# ---------------------------------------------------------------------------- -push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\ - $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1))) - -# ---------------------------------------------------------------------------- -# Function: pop -# Arguments: 1: Name of stack -# Returns: Top element from the stack after removing it -# ---------------------------------------------------------------------------- -pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \ - $(call first,$(__gmsl_stack_$1)) \ - $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1))))) - -# ---------------------------------------------------------------------------- -# Function: peek -# Arguments: 1: Name of stack -# Returns: Top element from the stack without removing it -# ---------------------------------------------------------------------------- -peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1)) - -# ---------------------------------------------------------------------------- -# Function: depth -# Arguments: 1: Name of stack -# Returns: Number of items on the stack -# ---------------------------------------------------------------------------- -depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1)) - -endif # __gmsl_have_eval - -ifdef __gmsl_have_eval -# ########################################################################### -# STRING CACHE -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: memoize -# Arguments: 1. Name of the function to be called if the string -# has not been previously seen -# 2. A string -# Returns: Returns the result of a memo function (which the user must -# define) on the passed in string and remembers the result. -# -# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache -# of MD5 hashes of strings. $(call memoize,memo,foo bar baz) -# ---------------------------------------------------------------------------- -__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2) -memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\ - $(call get,__gmsl_m,$(__gmsl_memoize)), \ - $(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2)))) - -endif # __gmsl_have_eval - -# ########################################################################### -# DEBUGGING FACILITIES -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Target: gmsl-echo-% -# Arguments: The % should be replaced by the name of a variable that you -# wish to print out. -# Action: Echos the value of the variable that matches the %. -# For example, 'make gmsl-echo-SHELL' will output the value of -# the SHELL variable. -# ---------------------------------------------------------------------------- -gmsl-echo-%: ; @echo $($*) - -# ---------------------------------------------------------------------------- -# Target: gmsl-print-% -# Arguments: The % should be replaced by the name of a variable that you -# wish to print out. -# Action: Echos the name of the variable that matches the % and its value. -# For example, 'make gmsl-print-SHELL' will output the value of -# the SHELL variable -# ---------------------------------------------------------------------------- -gmsl-print-%: ; @echo $* = $($*) - -# ---------------------------------------------------------------------------- -# Function: assert -# Arguments: 1: A boolean that must be true or the assertion will fail -# 2: The message to print with the assertion -# Returns: None -# ---------------------------------------------------------------------------- -assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2))) - -# ---------------------------------------------------------------------------- -# Function: assert_exists -# Arguments: 1: Name of file that must exist, if it is missing an assertion -# will be generated -# Returns: None -# ---------------------------------------------------------------------------- -assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing)) - -# ---------------------------------------------------------------------------- -# Function: assert_no_dollar -# Arguments: 1: Name of a function being executd -# 2: Arguments to check -# Returns: None -# ---------------------------------------------------------------------------- -assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument) - -# ---------------------------------------------------------------------------- -# Function: assert_no_space -# Arguments: 1: Name of a function being executd -# 2: Arguments to check -# Returns: None -# ---------------------------------------------------------------------------- -ifeq ($(__gmsl_spaced_vars),$(false)) -assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument) -else -assert_no_space = -endif diff --git a/mk/gmsl b/mk/gmsl deleted file mode 100644 index b22949a31cf..00000000000 --- a/mk/gmsl +++ /dev/null @@ -1,85 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2022 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# Determine if the library has already been included and if so don't -# bother including it again - -ifndef __gmsl_included - -# Standard definitions for true and false. true is any non-empty -# string, false is an empty string. These are intended for use with -# $(if). - -true := T -false := - -# ---------------------------------------------------------------------------- -# Function: not -# Arguments: 1: A boolean value -# Returns: Returns the opposite of the arg. (true -> false, false -> true) -# ---------------------------------------------------------------------------- -not = $(if $1,$(false),$(true)) - -# Prevent reinclusion of the library - -__gmsl_included := $(true) - -# Try to determine where this file is located. If the caller did -# include /foo/gmsl then extract the /foo/ so that __gmsl gets -# included transparently - -__gmsl_root := - -ifneq ($(MAKEFILE_LIST),) -__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) - -# If there are any spaces in the path in __gmsl_root then give up - -ifeq (1,$(words $(__gmsl_root))) -__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root)) -endif - -endif - -include $(__gmsl_root)__gmsl - -endif # __gmsl_included - diff --git a/mk/gmsl.html b/mk/gmsl.html deleted file mode 100644 index a8da4698752..00000000000 --- a/mk/gmsl.html +++ /dev/null @@ -1,733 +0,0 @@ - - - - GNU Make Standard Library - - -

GNU Make Standard Library

-The GNU Make Standard Library (GMSL) is a collection of functions -implemented using native GNU Make functionality that provide list and -string manipulation, integer arithmetic, associative arrays, stacks, -and debugging facilities.  The GMSL is released under the BSD License.
-
-[Project Page] -[Releases] -
-

Using GMSL

-The two files needed are gmsl -and __gmsl.  To -include the GMSL in your Makefile do
-
include gmsl
-gmsl automatically includes __gmsl.  To check that -you have the right version of gmsl -use the gmsl_compatible -function (see -below). The current version is 1 2 0.
-
-The GMSL package also includes a test suite for GMSL.  Just run make -f gmsl-tests.
-

Logical Operators

GMSL has boolean $(true) (a non-empty string) -and $(false) (an empty string).  The following operators can be -used with those variables.
-
-
not
- -
- -Arguments: A boolean value
- -Returns:   Returns $(true) if the boolean is $(false) and vice versa
- -
and
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if both of the booleans are true
-
or
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if either of the booleans is true
-
xor
-
-Arguments: Two boolean values
-Returns:   Returns $(true) if exactly one of the booleans is true
-
nand
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not and'
-
nor
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not or'
-
xnor
-
-Arguments: Two boolean values
-Returns:   Returns value of 'not xor'
-
-

List Manipulation Functions

- A list is a string of characters; the list separator is a space.
- -
-
first
-
-Arguments: 1: A list
-Returns:   Returns the first element of a list
- -
last
-
-Arguments: 1: A list
-Returns:   Returns the last element of a list
- -
rest
-
-Arguments: 1: A list
-Returns:   Returns the list with the first element -removed
- -
chop
-
-Arguments: 1: A list
-Returns:   Returns the list with the last element removed
- -
map
-
-Arguments: 1: Name of function to -$(call) for each element of list
-           2: List to -iterate over calling the function in 1
-Returns:   The list after calling the function on each -element
- -
pairmap
-
-Arguments: 1: Name of function to -$(call) for each pair of elements
-           2: List to -iterate over calling the function in 1
-           3: Second -list to iterate over calling the function in 1
-Returns:   The list after calling the function on each -pair of elements
- -
leq
-
-Arguments: 1: A list to compare -against...
-           2: ...this -list
-Returns:   Returns $(true) if the two lists are identical
- -
lne
-
-Arguments: 1: A list to compare -against...
-           2: ...this -list
-Returns:   Returns $(true) if the two lists are different
- -
reverse
-
-Arguments: 1: A list to reverse
-Returns:   The list with its elements in reverse order
- -
uniq
-
-Arguments: 1: A list to deduplicate
-Returns:   The list with elements in order without duplicates
- -
length
-
-Arguments: 1: A list
-Returns:   The number of elements in the list
- -
-

String Manipulation Functions

-A string is any sequence of characters.
-
-
seq
-
-Arguments: 1: A string to compare -against...
-           2: ...this -string
-Returns:   Returns $(true) if the two strings are -identical
- -
sne
-
-Arguments: 1: A string to compare -against...
-           2: ...this -string
-Returns:   Returns $(true) if the two strings are not -the same
- -
strlen
-
-Arguments: 1: A string
-Returns:   Returns the length of the string
- -
substr
-
-Arguments: 1: A string
-           2: Start offset (first character is 1)
-           3: Ending offset (inclusive)
Returns:   Returns a substring
- -
split
-
-Arguments: 1: The character to -split on
-           2: A -string to split
-Returns:   Splits a string into a list separated by -spaces at the split
-           character -in the first argument
- -
merge
-
-Arguments: 1: The character to -put between fields
-           2: A list -to merge into a string
-Returns:   Merges a list into a single string, list -elements are separated
-           by the -character in the first argument
- -
tr
-
-Arguments: 1: The list of -characters to translate from
-           2: The -list of characters to translate to
-           3: The -text to translate
-Returns:   Returns the text after translating characters
- -
uc
-
-Arguments: 1: Text to upper case
-Returns:   Returns the text in upper case
- -
lc
-
-Arguments: 1: Text to lower case
-Returns:   Returns the text in lower case
- -
-

Set Manipulation Functions

-Sets are represented by sorted, deduplicated lists. To create a set -from a list use set_create, or start with the empty_set and set_insert individual elements. -The empty set is defined as empty_set.

- -

set_create
-
-Arguments: 1: A list of set elements
-Returns:   Returns the newly created set
- - -
set_insert
-
-Arguments: 1: A single element to add to a set
-           2: A set
-Returns:   Returns the set with the element added
- - -
set_remove
-
-Arguments: 1: A single element to remove from a set
-           2: A set
-Returns:   Returns the set with the element removed
- - -
set_is_member
-
-Arguments: 1: A single element
-           2: A set
-Returns:   Returns $(true) if the element is in the set
- - -
set_is_not_member
-
-Arguments: 1: A single element
-           2: A set
-Returns:   Returns $(false) if the element is in the set
- - -
set_union
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns the union of the two sets
- - -
set_intersection
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns the intersection of the two sets
- - -
set_is_subset
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns $(true) if the first set is a subset of the second
- - -
set_equal
-
-Arguments: 1: A set
-           2: Another set
-Returns:   Returns $(true) if the two sets are identical
- - -
-

Integer Arithmetic Functions

-Integers are represented by lists with the equivalent number of -x's.  For example the number 4 is x x x x.  The maximum -integer that the library can handle as input (i.e. as the argument to a -call to int_encode) is -65536. There is no limit on integer size for internal computations or -output.
-
-The arithmetic library functions come in two forms: one form of each -function takes integers as arguments and the other form takes the -encoded form (x's created by a call to int_encode).  For example, -there are two plus functions: plus -(called with integer arguments and returns an integer) and int_plus (called with encoded -arguments and returns an encoded result).
-
-plus will be slower than int_plus because its arguments -and result have to be translated between the x's format and -integers.  If doing a complex calculation use the int_* forms with a single -encoding of inputs and single decoding of the output.  For simple -calculations the direct forms can be used.
-
-
int_decode
-
-Arguments: 1: A number of x's -representation
-Returns:   Returns the integer for human consumption -that is represented
-           by the -string of x's
- -
int_encode
-
-Arguments: 1: A number in -human-readable integer form
-Returns:   Returns the integer encoded as a string of x's
- -
int_plus
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the sum of the two numbers in x's -representation
- -
plus (wrapped version of int_plus)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the sum of the two integers
- -
int_subtract
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the difference of the two numbers in -x's representation,
-           or outputs -an error on a numeric underflow
- -
subtract (wrapped version of int_subtract)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the difference of the two integers,
-           or outputs -an error on a numeric underflow
- -
int_multiply
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the product of the two numbers in x's -representation
- -
multiply (wrapped version of int_multiply)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the product of the two integers
- -
int_divide
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the result of integer division of -argument 1 divided
-           by -argument 2 in x's representation
- -
divide (wrapped version of int_divide)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the integer division of the first -argument by the second
- -
int_modulo
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the remainder of integer division of -argument 1 divided
-           by -argument 2 in x's representation
- -
modulo (wrapped version of int_modulo)
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the remainder of integer division of the first -argument by the second
- -
int_max, int_min
-
-Arguments: 1: A number in x's -representation
-           2: Another -number in x's represntation
-Returns:   Returns the maximum or minimum of its -arguments in x's
-           -representation
- -
max, min
-
-Arguments: 1: An integer
-           2: Another -integer
-Returns:   Returns the maximum or minimum of its integer -arguments
- -
int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
-
-Arguments: Two x's representation -numbers to be compared
-Returns:   $(true) or $(false)
-
-int_gt First argument greater than second argument
-int_gte First argument greater than or equal to second argument
-int_lt First argument less than second argument
-int_lte First argument less than or equal to second argument
-int_eq First argument is numerically equal to the second argument
-int_ne First argument is not numerically equal to the second argument
- -
gt, gte, lt, lte, eq, ne
-
-Arguments: Two integers to be -compared
-Returns:   $(true) or $(false)
-
-gt First argument greater than second argument
-gte First argument greater than or equal to second argument
-lt First argument less than second argument
-lte First argument less than or equal to second argument
-eq First argument is numerically equal to the second argument
-ne First argument is not numerically equal to the second argument
- -increment adds 1 to its argument, decrement subtracts 1. Note that
-decrement does not range check and hence will not underflow, but
-will incorrectly say that 0 - 1 = 0
-
int_inc
-
-Arguments: 1: A number in x's -representation
-Returns:   The number incremented by 1 in x's -representation
- -
inc
-
-Arguments: 1: An integer
-Returns:   The argument incremented by 1
- -
int_dec
-
-Arguments: 1: A number in x's -representation
-Returns:   The number decremented by 1 in x's -representation
- -
dec
-
-Arguments: 1: An integer
-Returns:   The argument decremented by 1
- -
int_double
-
-Arguments: 1: A number in x's -representation
-Returns:   The number doubled (i.e. * 2) and returned in -x's representation
- -
double
-
-Arguments: 1: An integer
-Returns:   The integer times 2
- -
int_halve
-
-Arguments: 1: A number in x's -representation
-Returns:   The number halved (i.e. / 2) and returned in -x's representation
- -
halve
-
-Arguments: 1: An integer
-Returns:   The integer divided by 2
- -
sequence
-
-Arguments: 1: An integer
-           2: An integer
-Returns:   The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1
- -
dec2hex, dec2bin, dec2oct
-
-Arguments: 1: An integer
-Returns:   The decimal argument converted to hexadecimal, binary or octal
- -
-

Associative Arrays

-An associate array maps a key value (a string with no spaces in it) to -a single value (any string).   
-
- -
set
-
-Arguments: 1: Name of associative -array
-           2: The key -value to associate
-           3: The -value associated with the key
-Returns:   Nothing
- -
get
-
-Arguments: 1: Name of associative -array
-           2: The key -to retrieve
-Returns:   The value stored in the array for that key
- -
keys
-
-Arguments: 1: Name of associative -array
-Returns:   Returns a list of all defined keys in the -array
- -
defined
-
-Arguments: 1: Name of associative -array
-           2: The key -to test
-Returns:   Returns true if the key is defined (i.e. not -empty)
- -
-

Named Stacks

-A stack is an ordered list of strings (with no spaces in them).
-
-
push
-
-Arguments: 1: Name of stack
-           2: Value -to push onto the top of the stack (must not contain
-           a space)
-Returns:   None
- -
pop
-
-Arguments: 1: Name of stack
-Returns:   Top element from the stack after removing it
- -
peek
-
-Arguments: 1: Name of stack
-Returns:   Top element from the stack without removing it
- -
depth
-
-Arguments: 1: Name of stack
-Returns:   Number of items on the stack
- -
-

Function memoization

-To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.
-
-
memoize
-
-Arguments: 1: Name of function to memoize
-           2: String argument for the function
-Returns:   Result of $1 applied to $2 but only calls $1 once for each unique $2
- - -
-

Miscellaneous and Debugging Facilities

-GMSL defines the following constants; all are accessed as normal GNU -Make variables by wrapping them in $() or ${}.
-
- - - - - - - - - - - - - - - - - - - - - - - -
Constant
- 		Value
- 		Purpose
-
true
- 		T
- 		Boolean for $(if) -and return from  GMSL functions
-
false
-
- 		Boolean for $(if) -and return from GMSL functions
-
gmsl_version
- 		1 0 0
- 		GMSL version number as list: major minor revision
-
-
-gmsl_compatible
-
-Arguments: List containing the desired library version number (maj min -rev)
-Returns:   -$(true) if this version of the library is compatible
-           -with the requested version number, otherwise $(false) -
gmsl-print-% (target not a function)
-
-Arguments: The % should be -replaced by the name of a variable that you
-           wish to -print out.
-Action:    Echos the name of the variable that matches -the % and its value.
-           For -example, 'make gmsl-print-SHELL' will output the value of
-           the SHELL -variable
- -
gmsl-echo-% (target not a function)
-
-Arguments: The % should be -replaced by the name of a variable that you
-           wish to -print out.
-Action:    Echos the value of the variable that matches -the %.
-           For -example, 'make gmsl-echo-SHELL' will output the value of
-           the SHELL -variable
- -
assert
-
-Arguments: 1: A boolean that must -be true or the assertion will fail
-           2: The -message to print with the assertion
-Returns:   None
- -
assert_exists
-
-Arguments: 1: Name of file that -must exist, if it is missing an assertion
-           will be -generated
-Returns:   None
- -

-GMSL has a number of environment variables (or command-line overrides) -that control various bits of functionality:
-
- - - - - - - - - - - - - - - - - - - -
Variable
- 		Purpose
-
GMSL_NO_WARNINGS
- 		If set prevents GMSL from outputting warning messages: -artithmetic functions generate underflow warnings.
-
GMSL_NO_ERRORS
- 		If set prevents GMSL from generating fatal errors: division -by zero or failed assertions are fatal.
-
GMSL_TRACE
- 		Enables function tracing.  Calls to GMSL functions will -result in name and arguments being traced.
-
-
-
-Copyright (c) 2005-2022 John Graham-Cumming.
-
- diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index 8f06c93284f..f611693f48b 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -7,14 +7,8 @@ MKDIR=mkdir -p # Go should not be required to run functional tests GOOS ?= $(shell go env GOOS) -# Current versioning information from env +#Current versioning information from env BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data - -PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null) - -# See if we have libre2-dev installed for C++ optimizations. -# In fedora and other distros, we need to tell where to find re2.pc -RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index 68abc68537b..8e2cdf19b40 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -4,9 +4,9 @@ MAKE=make GOOS=windows PREFIX=$(shell $$env:TEMP) -# Current versioning information from env -# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') -# hardcode it till I find a workaround +#Current versioning information from env +#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') +#hardcode it till i find a workaround BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1)) BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss") DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config @@ -18,5 +18,3 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 - -PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index a7916dc2bcf..9fc8fc86f72 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -35,20 +35,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -type DataSourceUnavailableError struct { - Name string - Err error -} - -func (e *DataSourceUnavailableError) Error() string { - return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err) -} - -func (e *DataSourceUnavailableError) Unwrap() error { - return e.Err -} - - // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -89,10 +75,6 @@ func GetDataSourceIface(dataSourceType string) DataSource { return source() } -// DataSourceConfigure creates and returns a DataSource object from a configuration, -// if the configuration is not valid it returns an error. -// If the datasource can't be run (eg. journalctl not available), it still returns an error which -// can be checked for the appropriate action. func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { // we dump it back to []byte, because we want to decode the yaml blob twice: // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource @@ -118,7 +100,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} + return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err) } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { @@ -191,11 +173,10 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) - idx := -1 for { var sub configuration.DataSourceCommonCfg + var idx int err = dec.Decode(&sub) - idx += 1 if err != nil { if !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) @@ -212,6 +193,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) + idx += 1 continue } return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) @@ -226,11 +208,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, sub.UniqueId = uniqueId src, err := DataSourceConfigure(sub) if err != nil { - var dserr *DataSourceUnavailableError - if errors.As(err, &dserr) { - log.Error(err) - continue - } return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err) } if sub.TransformExpr != "" { @@ -241,6 +218,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, transformRuntimes[uniqueId] = vm } sources = append(sources, *src) + idx += 1 } } return sources, nil @@ -317,11 +295,6 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { - // Don't wait if we have no sources, as it will hang forever - if len(sources) == 0 { - return nil - } - for i := 0; i < len(sources); i++ { subsrc := sources[i] //ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) @@ -357,8 +330,11 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } - - /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() - return err + // Don't wait if we have no sources, as it will hang forever + if len(sources) > 0 { + /*return only when acquisition is over (cat) or never (tail)*/ + err := AcquisTomb.Wait() + return err + } + return nil } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 548ecc04bb5..6b6d5ce7106 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -167,7 +167,7 @@ log_level: debug source: mock_cant_run wowo: ajsajasjas `, - ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro", + ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", }, } diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 48bbe421753..1abf04c5b35 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -200,7 +201,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro targetStream := "*" if cw.Config.StreamRegexp != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { - return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err) + return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) } targetStream = *cw.Config.StreamRegexp } else if cw.Config.StreamName != nil { @@ -344,7 +345,8 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig }, ) if err != nil { - return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) + newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName) + return newerr } cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } @@ -493,7 +495,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) + newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName) cfg.logger.Warningf("err : %s", newerr) return newerr } @@ -530,7 +532,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, u, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("while parsing %s: %w", dsn, err) + return errors.Wrapf(err, "while parsing %s", dsn) } for k, v := range u { @@ -541,7 +543,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(v[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", v[0], err) + return errors.Wrapf(err, "unknown level %s", v[0]) } cw.logger.Logger.SetLevel(lvl) @@ -575,7 +577,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, //let's reuse our parser helper so that a ton of date formats are supported duration, err := time.ParseDuration(v[0]) if err != nil { - return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err) + return errors.Wrapf(err, "unable to parse '%s' as duration", v[0]) } cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration) start := time.Now().UTC().Add(-duration) @@ -672,7 +674,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err) + return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName) } cfg.logger.Tracef("after GetLogEventsPagesWithContext") case <-cw.t.Dying(): diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index 92962697437..b1808e446bb 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -12,6 +12,7 @@ import ( dockerTypes "github.com/docker/docker/api/types" "github.com/docker/docker/client" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -79,7 +80,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &d.Config) if err != nil { - return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err) + return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") } if d.logger != nil { @@ -213,7 +214,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg parameters, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("while parsing parameters %s: %w", dsn, err) + return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err) } for k, v := range parameters { @@ -224,7 +225,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg } lvl, err := log.ParseLevel(v[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", v[0], err) + return errors.Wrapf(err, "unknown level %s", v[0]) } d.logger.Logger.SetLevel(lvl) case "until": diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 799e78dca3d..c24b1733218 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/fsnotify/fsnotify" "github.com/nxadm/tail" "github.com/pkg/errors" @@ -22,8 +24,6 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { f.watcher, err = fsnotify.NewWatcher() if err != nil { - return fmt.Errorf("could not create fsnotify watcher: %w", err) + return errors.Wrapf(err, "Could not create fsnotify watcher") } f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) @@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { } files, err := filepath.Glob(pattern) if err != nil { - return fmt.Errorf("glob failure: %w", err) + return errors.Wrap(err, "Glob failure") } if len(files) == 0 { f.logger.Warnf("No matching files for pattern %s", pattern) @@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("could not parse file args: %w", err) + return errors.Wrap(err, "could not parse file args") } for key, value := range params { switch key { @@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } f.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err) + return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0]) } f.config.MaxBufferSize = maxBufferSize default: @@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger f.logger.Debugf("Will try pattern %s", args[0]) files, err := filepath.Glob(args[0]) if err != nil { - return fmt.Errorf("glob failure: %w", err) + return errors.Wrap(err, "Glob failure") } if len(files) == 0 { @@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): err := f.watcher.Close() if err != nil { - return fmt.Errorf("could not remove all inotify watches: %w", err) + return errors.Wrapf(err, "could not remove all inotify watches") } return nil } @@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom fd, err := os.Open(filename) if err != nil { - return fmt.Errorf("failed opening %s: %w", filename, err) + return errors.Wrapf(err, "failed opening %s", filename) } defer fd.Close() @@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom gz, err := gzip.NewReader(fd) if err != nil { logger.Errorf("Failed to read gz file: %s", err) - return fmt.Errorf("failed to read gz %s: %w", filename, err) + return errors.Wrapf(err, "failed to read gz %s", filename) } defer gz.Close() scanner = bufio.NewScanner(gz) diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index 8f80d050fd9..ff55bc4134e 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) { { name: "glob syntax error", config: `filename: "[asd-.log"`, - expectedErr: "glob failure: syntax error in pattern", + expectedErr: "Glob failure: syntax error in pattern", }, { name: "bad exclude regexp", @@ -150,7 +150,7 @@ filename: /`, config: ` mode: cat filename: "[*-.log"`, - expectedConfigErr: "glob failure: syntax error in pattern", + expectedConfigErr: "Glob failure: syntax error in pattern", logLevel: log.WarnLevel, expectedLines: 0, }, diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index b060ac364c4..7882cb7c29c 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -236,7 +237,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } j.logger.Logger.SetLevel(lvl) case "since": diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index dba8daf7592..085751cfc59 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/segmentio/kafka-go" log "github.com/sirupsen/logrus" @@ -92,12 +93,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error { dialer, err := k.Config.NewDialer() if err != nil { - return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err) + return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) } k.Reader, err = k.Config.NewReader(dialer) if err != nil { - return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err) + return errors.Wrapf(err, "cannote create %s reader", dataSourceName) } if k.Reader == nil { @@ -148,7 +149,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error { if err == io.EOF { return nil } - k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) + k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName)) } l := types.Line{ Raw: string(m.Value), @@ -180,7 +181,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic) if err := k.Reader.Close(); err != nil { - return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err) + return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic) } return nil } @@ -263,7 +264,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er rConf.GroupID = kc.GroupID } if err := rConf.Validate(); err != nil { - return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err) + return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration") } return kafka.NewReader(rConf), nil } diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index 950d33a62e2..b37d0e7b7a7 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -8,14 +8,13 @@ import ( "testing" "time" - "github.com/segmentio/kafka-go" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/segmentio/kafka-go" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gotest.tools/v3/assert" ) func TestConfigure(t *testing.T) { @@ -179,7 +178,7 @@ topic: crowdsecplaintext`), subLogger) break READLOOP } } - require.Equal(t, ts.expectedLines, actualLines) + assert.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) @@ -255,7 +254,7 @@ tls: break READLOOP } } - require.Equal(t, ts.expectedLines, actualLines) + assert.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index cc263da4f7d..60cdc3751e7 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -13,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/kinesis" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -123,7 +124,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { - return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err) + return errors.Wrap(err, "Cannot parse kinesis datasource configuration") } if k.Config.Mode == "" { @@ -217,7 +218,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea return nil default: k.logger.Errorf("Error while waiting for consumer deregistration: %s", err) - return fmt.Errorf("cannot describe stream consumer: %w", err) + return errors.Wrap(err, "Cannot describe stream consumer") } } time.Sleep(time.Millisecond * 200 * time.Duration(i+1)) @@ -235,12 +236,12 @@ func (k *KinesisSource) DeregisterConsumer() error { switch err.(type) { case *kinesis.ResourceNotFoundException: default: - return fmt.Errorf("cannot deregister stream consumer: %w", err) + return errors.Wrap(err, "Cannot deregister stream consumer") } } err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN) if err != nil { - return fmt.Errorf("cannot wait for consumer deregistration: %w", err) + return errors.Wrap(err, "Cannot wait for consumer deregistration") } return nil } @@ -252,7 +253,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error { ConsumerARN: aws.String(consumerARN), }) if err != nil { - return fmt.Errorf("cannot describe stream consumer: %w", err) + return errors.Wrap(err, "Cannot describe stream consumer") } if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" { k.logger.Debugf("Consumer %s is active", consumerARN) @@ -271,11 +272,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu StreamARN: aws.String(k.Config.StreamARN), }) if err != nil { - return nil, fmt.Errorf("cannot register stream consumer: %w", err) + return nil, errors.Wrap(err, "Cannot register stream consumer") } err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN) if err != nil { - return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err) + return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active") } return streamConsumer, nil } @@ -338,7 +339,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven logger.Infof("Subscribed shard reader is dying") err := reader.Close() if err != nil { - return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err) + return errors.Wrap(err, "Cannot close kinesis subscribed shard reader") } return nil case event, ok := <-reader.Events(): @@ -361,7 +362,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R StreamName: aws.String(arn.Resource[7:]), }) if err != nil { - return fmt.Errorf("cannot list shards for enhanced_read: %w", err) + return errors.Wrap(err, "Cannot list shards for enhanced_read") } for _, shard := range shards.Shards { @@ -372,7 +373,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R ConsumerARN: streamConsumer.Consumer.ConsumerARN, }) if err != nil { - return fmt.Errorf("cannot subscribe to shard: %w", err) + return errors.Wrap(err, "Cannot subscribe to shard") } k.shardReaderTomb.Go(func() error { return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:]) @@ -384,7 +385,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { parsedARN, err := arn.Parse(k.Config.StreamARN) if err != nil { - return fmt.Errorf("cannot parse stream ARN: %w", err) + return errors.Wrap(err, "Cannot parse stream ARN") } if !strings.HasPrefix(parsedARN.Resource, "stream/") { return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN) @@ -394,12 +395,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { k.logger.Info("starting kinesis acquisition with enhanced fan-out") err = k.DeregisterConsumer() if err != nil { - return fmt.Errorf("cannot deregister consumer: %w", err) + return errors.Wrap(err, "Cannot deregister consumer") } streamConsumer, err := k.RegisterConsumer() if err != nil { - return fmt.Errorf("cannot register consumer: %w", err) + return errors.Wrap(err, "Cannot register consumer") } for { @@ -407,7 +408,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { err = k.SubscribeToShards(parsedARN, streamConsumer, out) if err != nil { - return fmt.Errorf("cannot subscribe to shards: %w", err) + return errors.Wrap(err, "Cannot subscribe to shards") } select { case <-t.Dying(): @@ -416,7 +417,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { _ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves err = k.DeregisterConsumer() if err != nil { - return fmt.Errorf("cannot deregister consumer: %w", err) + return errors.Wrap(err, "Cannot deregister consumer") } return nil case <-k.shardReaderTomb.Dying(): @@ -439,7 +440,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)}) if err != nil { logger.Errorf("Cannot get shard iterator: %s", err) - return fmt.Errorf("cannot get shard iterator: %w", err) + return errors.Wrap(err, "Cannot get shard iterator") } it := sharIt.ShardIterator //AWS recommends to wait for a second between calls to GetRecords for a given shard @@ -460,7 +461,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro continue default: logger.Error("Cannot get records") - return fmt.Errorf("cannot get records: %w", err) + return errors.Wrap(err, "Cannot get records") } } k.ParseAndPushRecords(records.Records, out, logger, shardId) @@ -485,7 +486,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error StreamName: aws.String(k.Config.StreamName), }) if err != nil { - return fmt.Errorf("cannot list shards: %w", err) + return errors.Wrap(err, "Cannot list shards") } k.shardReaderTomb = &tomb.Tomb{} for _, shard := range shards.Shards { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 24354738114..f65a0aa579a 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -8,16 +8,16 @@ import ( "net/http" "strings" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" "k8s.io/apiserver/pkg/apis/audit" - - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type KubernetesAuditConfiguration struct { @@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { k8sConfig := KubernetesAuditConfiguration{} err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig) if err != nil { - return fmt.Errorf("cannot parse k8s-audit configuration: %w", err) + return errors.Wrap(err, "Cannot parse k8s-audit configuration") } ka.config = k8sConfig @@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t t.Go(func() error { err := ka.server.ListenAndServe() if err != nil && err != http.ErrServerClosed { - return fmt.Errorf("k8s-audit server failed: %w", err) + return errors.Wrap(err, "k8s-audit server failed") } return nil }) diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 96026f55e69..4ba31f43b58 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -6,7 +6,6 @@ import ( "compress/gzip" "context" "encoding/json" - "errors" "fmt" "io" "net/url" @@ -22,13 +21,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type S3Configuration struct { @@ -564,7 +563,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return fmt.Errorf("could not parse s3 args: %w", err) + return errors.Wrap(err, "could not parse s3 args") } for key, value := range params { switch key { @@ -574,7 +573,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } lvl, err := log.ParseLevel(value[0]) if err != nil { - return fmt.Errorf("unknown level %s: %w", value[0], err) + return errors.Wrapf(err, "unknown level %s", value[0]) } s.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -583,7 +582,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return fmt.Errorf("invalid value for 'max_buffer_size': %w", err) + return errors.Wrapf(err, "invalid value for 'max_buffer_size'") } s.logger.Debugf("Setting max buffer size to %d", maxBufferSize) s.Config.MaxBufferSize = maxBufferSize diff --git a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go index 7118c295b54..088ab0d95e1 100644 --- a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go +++ b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -30,18 +31,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error { s.port = port udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port)) if err != nil { - return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err) + return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr) } udpConn, err := net.ListenUDP("udp", udpAddr) if err != nil { - return fmt.Errorf("could not listen on port %d: %w", s.port, err) + return errors.Wrapf(err, "could not listen on port %d", s.port) } s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port) s.udpConn = udpConn err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond)) if err != nil { - return fmt.Errorf("could not set read deadline on UDP socket: %w", err) + return errors.Wrap(err, "could not set read deadline on UDP socket") } return nil } @@ -86,7 +87,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb { func (s *SyslogServer) KillServer() error { err := s.udpConn.Close() if err != nil { - return fmt.Errorf("could not close UDP connection: %w", err) + return errors.Wrap(err, "could not close UDP connection") } close(s.channel) return nil diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 840e372007b..948f3d005e8 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -99,7 +100,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &s.config) if err != nil { - return fmt.Errorf("cannot parse syslog configuration: %w", err) + return errors.Wrap(err, "Cannot parse syslog configuration") } if s.config.Addr == "" { @@ -139,7 +140,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) s.server.SetChannel(c) err := s.server.Listen(s.config.Addr, s.config.Port) if err != nil { - return fmt.Errorf("could not start syslog server: %w", err) + return errors.Wrap(err, "could not start syslog server") } s.serverTomb = s.server.StartServer() t.Go(func() error { diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index dd2ba2975ff..b8d1b7fc1aa 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - qs "github.com/google/go-querystring/query" - "github.com/crowdsecurity/crowdsec/pkg/models" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" ) // type ApiAlerts service @@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. u := fmt.Sprintf("%s/alerts", s.client.URLPrefix) params, err := qs.Values(opts) if err != nil { - return nil, nil, fmt.Errorf("building query: %w", err) + return nil, nil, errors.Wrap(err, "building query") } if len(params) > 0 { URI = fmt.Sprintf("%s?%s", u, params.Encode()) @@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. req, err := s.client.NewRequest(http.MethodGet, URI, nil) if err != nil { - return nil, nil, fmt.Errorf("building request: %w", err) + return nil, nil, errors.Wrap(err, "building request") } resp, err := s.client.Do(ctx, req, &alerts) if err != nil { - return nil, resp, fmt.Errorf("performing request: %w", err) + return nil, resp, errors.Wrap(err, "performing request") } return &alerts, resp, nil } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index aa5039f0bc7..f4ec8cabe2c 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -8,13 +8,12 @@ import ( "reflect" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAlertsListAsMachine(t *testing.T) { diff --git a/pkg/apiclient/auth.go b/pkg/apiclient/auth.go index 84df74456b1..48b971a061d 100644 --- a/pkg/apiclient/auth.go +++ b/pkg/apiclient/auth.go @@ -3,21 +3,23 @@ package apiclient import ( "bytes" "encoding/json" + "math/rand" + "sync" + "time" + + //"errors" "fmt" "io" - "math/rand" "net/http" "net/http/httputil" "net/url" - "sync" - "time" + "github.com/crowdsecurity/crowdsec/pkg/fflag" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/go-openapi/strfmt" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/fflag" - "github.com/crowdsecurity/crowdsec/pkg/models" + //"google.golang.org/appengine/log" ) type APIKeyTransport struct { @@ -167,11 +169,11 @@ func (t *JWTTransport) refreshJwtToken() error { enc.SetEscapeHTML(false) err = enc.Encode(auth) if err != nil { - return fmt.Errorf("could not encode jwt auth body: %w", err) + return errors.Wrap(err, "could not encode jwt auth body") } req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf) if err != nil { - return fmt.Errorf("could not create request: %w", err) + return errors.Wrap(err, "could not create request") } req.Header.Add("Content-Type", "application/json") client := &http.Client{ @@ -194,7 +196,7 @@ func (t *JWTTransport) refreshJwtToken() error { resp, err := client.Do(req) if err != nil { - return fmt.Errorf("could not get jwt token: %w", err) + return errors.Wrap(err, "could not get jwt token") } log.Debugf("auth-jwt : http %d", resp.StatusCode) @@ -215,10 +217,10 @@ func (t *JWTTransport) refreshJwtToken() error { } if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return fmt.Errorf("unable to decode response: %w", err) + return errors.Wrap(err, "unable to decode response") } if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil { - return fmt.Errorf("unable to parse jwt expiration: %w", err) + return errors.Wrap(err, "unable to parse jwt expiration") } t.Token = response.Token @@ -261,7 +263,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err != nil { /*we had an error (network error for example, or 401 because token is refused), reset the token ?*/ t.Token = "" - return resp, fmt.Errorf("performing jwt auth: %w", err) + return resp, errors.Wrapf(err, "performing jwt auth") } log.Debugf("resp-jwt: %d", resp.StatusCode) diff --git a/pkg/apiclient/auth_service.go b/pkg/apiclient/auth_service.go index 64284902e8c..26ad80c0c55 100644 --- a/pkg/apiclient/auth_service.go +++ b/pkg/apiclient/auth_service.go @@ -21,6 +21,7 @@ type enrollRequest struct { } func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) { + u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodDelete, u, nil) if err != nil { @@ -35,6 +36,7 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) } func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) { + u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, ®istration) diff --git a/pkg/apiclient/auth_service_test.go b/pkg/apiclient/auth_service_test.go index 32ba1890f62..6236cf0417b 100644 --- a/pkg/apiclient/auth_service_test.go +++ b/pkg/apiclient/auth_service_test.go @@ -10,12 +10,11 @@ import ( "net/url" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) type BasicMockPayload struct { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index d95f7749041..a7db1ed3380 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -11,6 +11,7 @@ import ( "net/url" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) var ( @@ -124,9 +125,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { /*if we have http status, return it*/ if err != nil { if resp != nil && resp.Response != nil { - return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err) + return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err) } - return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err) + return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err) } return c, nil @@ -165,7 +166,7 @@ func CheckResponse(r *http.Response) error { if err == nil && data != nil { err := json.Unmarshal(data, errorResponse) if err != nil { - return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err) + return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode) } } else { errorResponse.Message = new(string) diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index 9e082cf51cf..c5076904147 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/stretchr/testify/assert" ) func TestNewRequestInvalid(t *testing.T) { diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index 08f56730b86..ef52a60abda 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -9,9 +9,10 @@ import ( "runtime" "testing" - log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index e96394f5611..054c51a9ceb 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -6,15 +6,14 @@ import ( "fmt" "net/http" - qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" "github.com/crowdsecurity/crowdsec/pkg/types" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type DecisionsService service diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index ab7e46e644f..935ddcea599 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -8,15 +8,14 @@ import ( "reflect" "testing" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDecisionsList(t *testing.T) { diff --git a/pkg/apiclient/decisions_sync_service.go b/pkg/apiclient/decisions_sync_service.go index 57999691f21..e5284f4d9e7 100644 --- a/pkg/apiclient/decisions_sync_service.go +++ b/pkg/apiclient/decisions_sync_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type DecisionDeleteService service @@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix) req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions) if err != nil { - return nil, nil, fmt.Errorf("while building request: %w", err) + return nil, nil, errors.Wrap(err, "while building request") } resp, err := d.client.Do(ctx, req, &response) if err != nil { - return nil, resp, fmt.Errorf("while performing request: %w", err) + return nil, resp, errors.Wrap(err, "while performing request") } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Decisions delete response : http %s", resp.Response.Status) diff --git a/pkg/apiclient/signal.go b/pkg/apiclient/signal.go index 2dceb815754..27d2e3693e6 100644 --- a/pkg/apiclient/signal.go +++ b/pkg/apiclient/signal.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) type SignalService service @@ -18,12 +19,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque u := fmt.Sprintf("%s/signals", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, &signals) if err != nil { - return nil, nil, fmt.Errorf("while building request: %w", err) + return nil, nil, errors.Wrap(err, "while building request") } resp, err := s.client.Do(ctx, req, &response) if err != nil { - return nil, resp, fmt.Errorf("while performing request: %w", err) + return nil, resp, errors.Wrap(err, "while performing request") } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Signal push response : http %s", resp.Response.Status) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index b15cf21d6bd..3791a9a3c9b 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -33,8 +33,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -const ( - // delta values must be smaller than the interval +var ( pullIntervalDefault = time.Hour * 2 pullIntervalDelta = 5 * time.Minute pushIntervalDefault = time.Second * 10 @@ -72,12 +71,7 @@ type apic struct { // randomDuration returns a duration value between d-delta and d+delta func randomDuration(d time.Duration, delta time.Duration) time.Duration { - ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta - // ticker interval must be > 0 (nanoseconds) - if ret <= 0 { - return 1 - } - return ret + return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64())) } func (a *apic) FetchScenariosListFromDB() ([]string, error) { @@ -828,6 +822,80 @@ func (a *apic) Pull() error { } } +func (a *apic) GetMetrics() (*models.Metrics, error) { + metric := &models.Metrics{ + ApilVersion: ptr.Of(version.String()), + Machines: make([]*models.MetricsAgentInfo, 0), + Bouncers: make([]*models.MetricsBouncerInfo, 0), + } + machines, err := a.dbClient.ListMachines() + if err != nil { + return metric, err + } + bouncers, err := a.dbClient.ListBouncers() + if err != nil { + return metric, err + } + var lastpush string + for _, machine := range machines { + if machine.LastPush == nil { + lastpush = time.Time{}.String() + } else { + lastpush = machine.LastPush.String() + } + m := &models.MetricsAgentInfo{ + Version: machine.Version, + Name: machine.MachineId, + LastUpdate: machine.UpdatedAt.String(), + LastPush: lastpush, + } + metric.Machines = append(metric.Machines, m) + } + + for _, bouncer := range bouncers { + m := &models.MetricsBouncerInfo{ + Version: bouncer.Version, + CustomName: bouncer.Name, + Name: bouncer.Type, + LastPull: bouncer.LastPull.String(), + } + metric.Bouncers = append(metric.Bouncers, m) + } + return metric, nil +} + +func (a *apic) SendMetrics(stop chan (bool)) { + defer trace.CatchPanic("lapi/metricsToAPIC") + + ticker := time.NewTicker(a.metricsIntervalFirst) + + log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval) + + for { + metrics, err := a.GetMetrics() + if err != nil { + log.Errorf("unable to get metrics (%s), will retry", err) + } + _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) + if err != nil { + log.Errorf("capi metrics: failed: %s", err) + } else { + log.Infof("capi metrics: metrics sent successfully") + } + + select { + case <-stop: + return + case <-ticker.C: + ticker.Reset(a.metricsInterval) + case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? + a.pullTomb.Kill(nil) + a.pushTomb.Kill(nil) + return + } + } +} + func (a *apic) Shutdown() { a.pushTomb.Kill(nil) a.pullTomb.Kill(nil) diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go deleted file mode 100644 index 4befcf50ccb..00000000000 --- a/pkg/apiserver/apic_metrics.go +++ /dev/null @@ -1,145 +0,0 @@ -package apiserver - -import ( - "context" - "time" - - log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" - - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/models" -) - -func (a *apic) GetMetrics() (*models.Metrics, error) { - machines, err := a.dbClient.ListMachines() - if err != nil { - return nil, err - } - - machinesInfo := make([]*models.MetricsAgentInfo, len(machines)) - - for i, machine := range machines { - machinesInfo[i] = &models.MetricsAgentInfo{ - Version: machine.Version, - Name: machine.MachineId, - LastUpdate: machine.UpdatedAt.String(), - LastPush: ptr.OrEmpty(machine.LastPush).String(), - } - } - - bouncers, err := a.dbClient.ListBouncers() - if err != nil { - return nil, err - } - - bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers)) - - for i, bouncer := range bouncers { - bouncersInfo[i] = &models.MetricsBouncerInfo{ - Version: bouncer.Version, - CustomName: bouncer.Name, - Name: bouncer.Type, - LastPull: bouncer.LastPull.String(), - } - } - - return &models.Metrics{ - ApilVersion: ptr.Of(version.String()), - Machines: machinesInfo, - Bouncers: bouncersInfo, - }, nil -} - -func (a *apic) fetchMachineIDs() ([]string, error) { - machines, err := a.dbClient.ListMachines() - if err != nil { - return nil, err - } - - ret := make([]string, len(machines)) - for i, machine := range machines { - ret[i] = machine.MachineId - } - // sorted slices are required for the slices.Equal comparison - slices.Sort(ret) - return ret, nil -} - -// SendMetrics sends metrics to the API server until it receives a stop signal. -// -// Metrics are sent at start, then at the randomized metricsIntervalFirst, -// then at regular metricsInterval. If a change is detected in the list -// of machines, the next metrics are sent immediately. -func (a *apic) SendMetrics(stop chan (bool)) { - defer trace.CatchPanic("lapi/metricsToAPIC") - - // verify the list of machines every interval - const checkInt = 20 * time.Second - - // intervals must always be > 0 - metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval} - - log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", - metInts[1].Round(time.Second), metInts[2]) - - count := -1 - nextMetInt := func() time.Duration { - if count < len(metInts)-1 { - count++ - } - return metInts[count] - } - - // store the list of machine IDs to compare - // with the next list - machineIDs := []string{} - - reloadMachineIDs := func() { - ids, err := a.fetchMachineIDs() - if err != nil { - log.Debugf("unable to get machines (%s), will retry", err) - return - } - machineIDs = ids - } - - checkTicker := time.NewTicker(checkInt) - metTicker := time.NewTicker(nextMetInt()) - - for { - select { - case <-stop: - checkTicker.Stop() - metTicker.Stop() - return - case <-checkTicker.C: - oldIDs := machineIDs - reloadMachineIDs() - if !slices.Equal(oldIDs, machineIDs) { - log.Infof("capi metrics: machines changed, immediate send") - metTicker.Reset(1) - } - case <-metTicker.C: - metrics, err := a.GetMetrics() - if err != nil { - log.Errorf("unable to get metrics (%s), will retry", err) - } - log.Info("capi metrics: sending") - _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) - if err != nil { - log.Errorf("capi metrics: failed: %s", err) - } - metTicker.Reset(nextMetInt()) - case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? - checkTicker.Stop() - metTicker.Stop() - a.pullTomb.Kill(nil) - a.pushTomb.Kill(nil) - return - } - } -} diff --git a/pkg/apiserver/apic_metrics_test.go b/pkg/apiserver/apic_metrics_test.go deleted file mode 100644 index 7e37ea1e9d2..00000000000 --- a/pkg/apiserver/apic_metrics_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package apiserver - -import ( - "context" - "fmt" - "net/url" - "testing" - "time" - - "github.com/jarcoal/httpmock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/apiclient" -) - -func TestAPICSendMetrics(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedCalls int - setUp func(*apic) - metricsInterval time.Duration - }{ - { - name: "basic", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) {}, - }, - { - name: "with some metrics", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) { - api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) - api.dbClient.Ent.Machine.Create(). - SetMachineId("1234"). - SetPassword(testPassword.String()). - SetIpAddress("1.2.3.4"). - SetScenarios("crowdsecurity/test"). - SetLastPush(time.Time{}). - SetUpdatedAt(time.Time{}). - ExecX(context.Background()) - - api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) - api.dbClient.Ent.Bouncer.Create(). - SetIPAddress("1.2.3.6"). - SetName("someBouncer"). - SetAPIKey("foobar"). - SetRevoked(false). - SetLastPull(time.Time{}). - ExecX(context.Background()) - }, - }, - } - - httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) - httpmock.Activate() - defer httpmock.Deactivate() - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - url, err := url.ParseRequestURI("http://api.crowdsec.net/") - require.NoError(t, err) - - apiClient, err := apiclient.NewDefaultClient( - url, - "/api", - fmt.Sprintf("crowdsec/%s", version.String()), - nil, - ) - require.NoError(t, err) - - api := getAPIC(t) - api.pushInterval = time.Millisecond - api.pushIntervalFirst = time.Millisecond - api.apiClient = apiClient - api.metricsInterval = tc.metricsInterval - api.metricsIntervalFirst = tc.metricsInterval - tc.setUp(api) - - stop := make(chan bool) - httpmock.ZeroCallCounters() - go api.SendMetrics(stop) - time.Sleep(tc.duration) - stop <- true - - info := httpmock.GetCallCountInfo() - noResponderCalls := info["NO_RESPONDER"] - responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] - assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) - assert.Zero(t, noResponderCalls) - }) - } -} diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 8aeb092cd42..65ca299911b 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -1057,6 +1057,90 @@ func TestAPICPush(t *testing.T) { } } +func TestAPICSendMetrics(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedCalls int + setUp func(*apic) + metricsInterval time.Duration + }{ + { + name: "basic", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) {}, + }, + { + name: "with some metrics", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) { + api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) + api.dbClient.Ent.Machine.Create(). + SetMachineId("1234"). + SetPassword(testPassword.String()). + SetIpAddress("1.2.3.4"). + SetScenarios("crowdsecurity/test"). + SetLastPush(time.Time{}). + SetUpdatedAt(time.Time{}). + ExecX(context.Background()) + + api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) + api.dbClient.Ent.Bouncer.Create(). + SetIPAddress("1.2.3.6"). + SetName("someBouncer"). + SetAPIKey("foobar"). + SetRevoked(false). + SetLastPull(time.Time{}). + ExecX(context.Background()) + }, + }, + } + + httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) + httpmock.Activate() + defer httpmock.Deactivate() + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + url, err := url.ParseRequestURI("http://api.crowdsec.net/") + require.NoError(t, err) + + apiClient, err := apiclient.NewDefaultClient( + url, + "/api", + fmt.Sprintf("crowdsec/%s", version.String()), + nil, + ) + require.NoError(t, err) + + api := getAPIC(t) + api.pushInterval = time.Millisecond + api.pushIntervalFirst = time.Millisecond + api.apiClient = apiClient + api.metricsInterval = tc.metricsInterval + api.metricsIntervalFirst = tc.metricsInterval + tc.setUp(api) + + stop := make(chan bool) + httpmock.ZeroCallCounters() + go api.SendMetrics(stop) + time.Sleep(tc.duration) + stop <- true + + info := httpmock.GetCallCountInfo() + noResponderCalls := info["NO_RESPONDER"] + responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] + assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) + assert.Zero(t, noResponderCalls) + }) + } +} + func TestAPICPull(t *testing.T) { api := getAPIC(t) tests := []struct { diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 3facd0f6f29..c6510057366 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -9,18 +9,9 @@ import ( "net" "net/http" "os" - "path/filepath" "strings" "time" - "github.com/gin-gonic/gin" - "github.com/go-co-op/gocron" - "github.com/golang-jwt/jwt/v4" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" - "gopkg.in/tomb.v2" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -31,6 +22,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/gin-gonic/gin" + "github.com/go-co-op/gocron" + "github.com/golang-jwt/jwt/v4" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" + "gopkg.in/tomb.v2" ) var ( @@ -118,7 +116,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { logFile := "" if config.LogMedia == "file" { - logFile = filepath.Join(config.LogDir, "crowdsec_api.log") + logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir) } if log.GetLevel() < log.DebugLevel { @@ -164,7 +162,15 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.CompressLogs != nil { _compress = *config.CompressLogs } - + /*cf. https://github.com/natefinch/lumberjack/issues/82 + let's create the file beforehand w/ the right perms */ + // check if file exists + _, err := os.Stat(logFile) + // create file if not exists, purposefully ignore errors + if os.IsNotExist(err) { + file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600) + file.Close() + } LogOutput := &lumberjack.Logger{ Filename: logFile, MaxSize: _maxsize, //megabytes diff --git a/pkg/apiserver/controllers/v1/controller.go b/pkg/apiserver/controllers/v1/controller.go index 60da83d7dcb..3820bc10fd3 100644 --- a/pkg/apiserver/controllers/v1/controller.go +++ b/pkg/apiserver/controllers/v1/controller.go @@ -2,15 +2,17 @@ package v1 import ( "context" - "fmt" "net" + //"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers" + middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" "github.com/crowdsecurity/crowdsec/pkg/csprofiles" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" ) type Controller struct { @@ -46,7 +48,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) { profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg) if err != nil { - return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err) + return &Controller{}, errors.Wrapf(err, "failed to compile profiles") } v1 := &Controller{ diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index ce1bc8eeece..503f4d43d15 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -3,7 +3,7 @@ package v1 import ( "crypto/rand" "crypto/sha512" - "encoding/base64" + "encoding/hex" "fmt" "net/http" "strings" @@ -15,11 +15,9 @@ import ( log "github.com/sirupsen/logrus" ) -const ( +var ( APIKeyHeader = "X-Api-Key" bouncerContextKey = "bouncer_info" - // max allowed by bcrypt 72 = 54 bytes in base64 - dummyAPIKeySize = 54 ) type APIKey struct { @@ -33,7 +31,7 @@ func GenerateAPIKey(n int) (string, error) { if _, err := rand.Read(bytes); err != nil { return "", err } - return base64.StdEncoding.EncodeToString(bytes), nil + return hex.EncodeToString(bytes), nil } func NewAPIKey(dbClient *database.Client) *APIKey { @@ -84,7 +82,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { if err != nil && strings.Contains(err.Error(), "bouncer not found") { //Because we have a valid cert, automatically create the bouncer in the database if it does not exist //Set a random API key, but it will never be used - apiKey, err := GenerateAPIKey(dummyAPIKeySize) + apiKey, err := GenerateAPIKey(64) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/jwt.go b/pkg/apiserver/middlewares/v1/jwt.go index bbd33c54420..9f69f332e69 100644 --- a/pkg/apiserver/middlewares/v1/jwt.go +++ b/pkg/apiserver/middlewares/v1/jwt.go @@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { //Machine was not found, let's create it log.Printf("machine %s not found, create it", machineID) //let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli) - pwd, err := GenerateAPIKey(dummyAPIKeySize) + pwd, err := GenerateAPIKey(64) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/tls_auth.go b/pkg/apiserver/middlewares/v1/tls_auth.go index 87ca896a8f4..a0b837a41c6 100644 --- a/pkg/apiserver/middlewares/v1/tls_auth.go +++ b/pkg/apiserver/middlewares/v1/tls_auth.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ocsp" ) @@ -175,9 +176,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) ( } revoked, err := ta.isRevoked(cert, issuer) if err != nil { - //Fail securely, if we can't check the revocation status, let's consider the cert invalid + //Fail securely, if we can't check the revokation status, let's consider the cert invalid //We may change this in the future based on users feedback, but this seems the most sensible thing to do - return true, fmt.Errorf("could not check for client certification revocation status: %w", err) + return true, errors.Wrap(err, "could not check for client certification revokation status") } return revoked, nil @@ -230,7 +231,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) { revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1]) if err != nil { ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err) - return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err) + return false, "", errors.Wrap(err, "could not check for client certification revokation status") } if revoked { return false, "", fmt.Errorf("client certificate is revoked") diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 9d80cd95ac1..6add00c05df 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -38,7 +39,7 @@ func (c *Config) LoadCommon() error { } *k, err = filepath.Abs(*k) if err != nil { - return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) + return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) } } diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 24ff454b78d..59be93ae6c5 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -3,6 +3,8 @@ package csconfig import ( "fmt" "path/filepath" + + "github.com/pkg/errors" ) type ConfigurationPaths struct { @@ -48,7 +50,7 @@ func (c *Config) LoadConfigurationPaths() error { } *k, err = filepath.Abs(*k) if err != nil { - return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) + return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) } } diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index ec70fb459ae..41725bcf27d 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -2,13 +2,13 @@ package csconfig import ( "bytes" - "errors" "fmt" "io" "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/pkg/errors" "gopkg.in/yaml.v2" ) @@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error { if errors.Is(err, io.EOF) { break } - return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err) + return errors.Wrapf(err, "while decoding %s", c.ProfilesPath) } c.Profiles = append(c.Profiles, &t) } diff --git a/pkg/csplugin/broker.go b/pkg/csplugin/broker.go index b1bd54dfd53..6bc3b129639 100644 --- a/pkg/csplugin/broker.go +++ b/pkg/csplugin/broker.go @@ -2,7 +2,6 @@ package csplugin import ( "context" - "errors" "fmt" "io" "os" @@ -15,6 +14,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/google/uuid" plugin "github.com/hashicorp/go-plugin" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" @@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs pb.pluginProcConfig = pluginCfg pb.pluginsTypesToDispatch = make(map[string]struct{}) if err := pb.loadConfig(configPaths.NotificationDir); err != nil { - return fmt.Errorf("while loading plugin config: %w", err) + return errors.Wrap(err, "while loading plugin config") } if err := pb.loadPlugins(configPaths.PluginDir); err != nil { - return fmt.Errorf("while loading plugin: %w", err) + return errors.Wrap(err, "while loading plugin") } pb.watcher = PluginWatcher{} pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName) @@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error { data = []byte(csstring.StrictExpand(string(data), os.LookupEnv)) _, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data}) if err != nil { - return fmt.Errorf("while configuring %s: %w", pc.Name, err) + return errors.Wrapf(err, "while configuring %s", pc.Name) } log.Infof("registered plugin %s", pc.Name) pb.notificationPluginByName[pc.Name] = pluginClient @@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { parsedConfigs := make([]PluginConfig, 0) yamlFile, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("while opening %s: %w", path, err) + return parsedConfigs, errors.Wrapf(err, "while opening %s", path) } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) @@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { if errors.Is(err, io.EOF) { break } - return nil, fmt.Errorf("while decoding %s got error %s", path, err) + return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err) } // if the yaml document is empty, skip if reflect.DeepEqual(pc, PluginConfig{}) { diff --git a/pkg/csplugin/broker_suite_test.go b/pkg/csplugin/broker_suite_test.go index 6e3e51407ea..4c7cdd6eb42 100644 --- a/pkg/csplugin/broker_suite_test.go +++ b/pkg/csplugin/broker_suite_test.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) + type PluginSuite struct { suite.Suite @@ -22,19 +23,21 @@ type PluginSuite struct { // full path to the built plugin binary builtBinary string - runDir string // temporary directory for each test - pluginDir string // (config_paths.plugin_dir) - notifDir string // (config_paths.notification_dir) - pluginBinary string // full path to the plugin binary (unique for each test) - pluginConfig string // full path to the notification config (unique for each test) + runDir string // temporary directory for each test + pluginDir string // (config_paths.plugin_dir) + notifDir string // (config_paths.notification_dir) + pluginBinary string // full path to the plugin binary (unique for each test) + pluginConfig string // full path to the notification config (unique for each test) pluginBroker *PluginBroker } + func TestPluginSuite(t *testing.T) { suite.Run(t, new(PluginSuite)) } + func (s *PluginSuite) SetupSuite() { var err error @@ -54,12 +57,14 @@ func (s *PluginSuite) SetupSuite() { require.NoError(t, err, "while building dummy plugin") } + func (s *PluginSuite) TearDownSuite() { t := s.T() err := os.RemoveAll(s.buildDir) require.NoError(t, err) } + func copyFile(src string, dst string) error { s, err := os.Open(src) if err != nil { @@ -94,6 +99,7 @@ func (s *PluginSuite) TearDownTest() { s.TearDownSubTest() } + func (s *PluginSuite) SetupSubTest() { var err error t := s.T() @@ -119,7 +125,7 @@ func (s *PluginSuite) SetupSubTest() { require.NoError(t, err, "while copying built binary") err = os.Chmod(s.pluginBinary, 0o744) require.NoError(t, err, "chmod 0744 %s", s.pluginBinary) - + s.pluginConfig = path.Join(s.notifDir, "dummy.yaml") err = copyFile("testdata/dummy.yaml", s.pluginConfig) require.NoError(t, err, "while copying plugin config") diff --git a/pkg/csplugin/broker_test.go b/pkg/csplugin/broker_test.go index 991b89ed20c..467fadf4569 100644 --- a/pkg/csplugin/broker_test.go +++ b/pkg/csplugin/broker_test.go @@ -22,6 +22,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" ) + func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { return func(t *testing.T) { err := os.Chmod(s.pluginBinary, perm) @@ -29,28 +30,30 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { } } -func (s *PluginSuite) readconfig() PluginConfig { +func (s *PluginSuite) readconfig() (PluginConfig) { var config PluginConfig t := s.T() orig, err := os.ReadFile(s.pluginConfig) - require.NoError(t, err, "unable to read config file %s", s.pluginConfig) + require.NoError(t, err,"unable to read config file %s", s.pluginConfig) err = yaml.Unmarshal(orig, &config) - require.NoError(t, err, "unable to unmarshal config file") - + require.NoError(t, err,"unable to unmarshal config file") + return config } + func (s *PluginSuite) writeconfig(config PluginConfig) { t := s.T() data, err := yaml.Marshal(&config) - require.NoError(t, err, "unable to marshal config file") + require.NoError(t, err,"unable to marshal config file") err = os.WriteFile(s.pluginConfig, data, 0644) - require.NoError(t, err, "unable to write config file %s", s.pluginConfig) + require.NoError(t, err,"unable to write config file %s", s.pluginConfig) } + func (s *PluginSuite) TestBrokerInit() { tests := []struct { name string @@ -59,7 +62,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "group writable binary", @@ -346,7 +349,7 @@ func (s *PluginSuite) TestBrokerRunSimple() { DefaultEmptyTicker = 50 * time.Millisecond t := s.T() - + pb, err := s.InitBroker(nil) assert.NoError(t, err) diff --git a/pkg/csplugin/broker_win_test.go b/pkg/csplugin/broker_win_test.go index 01262b1fd0c..3d7498c0dd1 100644 --- a/pkg/csplugin/broker_win_test.go +++ b/pkg/csplugin/broker_win_test.go @@ -33,7 +33,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "no plugin dir", diff --git a/pkg/csplugin/listfiles.go b/pkg/csplugin/listfiles.go index c91be03b45d..2dea44f4fb1 100644 --- a/pkg/csplugin/listfiles.go +++ b/pkg/csplugin/listfiles.go @@ -13,9 +13,10 @@ func listFilesAtPath(path string) ([]string, error) { return nil, err } for _, file := range files { - if !file.IsDir() { + if ! file.IsDir() { filePaths = append(filePaths, filepath.Join(path, file.Name())) } } return filePaths, nil } + diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index 09102ef0da1..8bcedaa1fae 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -27,9 +27,9 @@ func TestListFilesAtPath(t *testing.T) { require.NoError(t, err) tests := []struct { - name string - path string - want []string + name string + path string + want []string expectedErr string }{ { @@ -41,8 +41,8 @@ func TestListFilesAtPath(t *testing.T) { }, }, { - name: "invalid directory", - path: "./foo/bar/", + name: "invalid directory", + path: "./foo/bar/", expectedErr: "open ./foo/bar/: " + cstest.PathNotFoundMessage, }, } diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go index 8ab1aa923b8..64a1e6e7184 100644 --- a/pkg/csplugin/notifier.go +++ b/pkg/csplugin/notifier.go @@ -4,10 +4,9 @@ import ( "context" "fmt" + "github.com/crowdsecurity/crowdsec/pkg/protobufs" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" - - "github.com/crowdsecurity/crowdsec/pkg/protobufs" ) type Notifier interface { diff --git a/pkg/csplugin/utils.go b/pkg/csplugin/utils.go index 5e19dee386f..67707c82955 100644 --- a/pkg/csplugin/utils.go +++ b/pkg/csplugin/utils.go @@ -3,7 +3,6 @@ package csplugin import ( - "errors" "fmt" "io/fs" "math" @@ -14,6 +13,8 @@ import ( "strconv" "strings" "syscall" + + "github.com/pkg/errors" ) func CheckCredential(uid int, gid int) *syscall.SysProcAttr { @@ -34,7 +35,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { } cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group) if err != nil { - return nil, fmt.Errorf("while getting process attributes: %w", err) + return nil, errors.Wrap(err, "while getting process attributes") } cmd.SysProcAttr.Credential.NoSetGroups = true } @@ -104,17 +105,17 @@ func pluginIsValid(path string) error { // check if it exists if details, err = os.Stat(path); err != nil { - return fmt.Errorf("plugin at %s does not exist: %w", path, err) + return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) } // check if it is owned by current user currentUser, err := user.Current() if err != nil { - return fmt.Errorf("while getting current user: %w", err) + return errors.Wrap(err, "while getting current user") } currentUID, err := getUID(currentUser.Username) if err != nil { - return fmt.Errorf("while looking up the current uid: %w", err) + return errors.Wrap(err, "while looking up the current uid") } stat := details.Sys().(*syscall.Stat_t) if stat.Uid != currentUID { diff --git a/pkg/csplugin/utils_windows.go b/pkg/csplugin/utils_windows.go index dfb11aff548..874e30021ab 100644 --- a/pkg/csplugin/utils_windows.go +++ b/pkg/csplugin/utils_windows.go @@ -13,6 +13,7 @@ import ( "syscall" "unsafe" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -53,38 +54,38 @@ func CheckPerms(path string) error { systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid)) if err != nil { - return fmt.Errorf("while creating SYSTEM well known sid: %w", err) + return errors.Wrap(err, "while creating SYSTEM well known sid") } adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid)) if err != nil { - return fmt.Errorf("while creating built-in Administrators well known sid: %w", err) + return errors.Wrap(err, "while creating built-in Administrators well known sid") } currentUser, err := user.Current() if err != nil { - return fmt.Errorf("while getting current user: %w", err) + return errors.Wrap(err, "while getting current user") } currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username) if err != nil { - return fmt.Errorf("while looking up current user sid: %w", err) + return errors.Wrap(err, "while looking up current user sid") } sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return fmt.Errorf("while getting owner security info: %w", err) + return errors.Wrap(err, "while getting owner security info") } if !sd.IsValid() { - return fmt.Errorf("security descriptor is invalid") + return errors.New("security descriptor is invalid") } owner, _, err := sd.Owner() if err != nil { - return fmt.Errorf("while getting owner: %w", err) + return errors.Wrap(err, "while getting owner") } if !owner.IsValid() { - return fmt.Errorf("owner is invalid") + return errors.New("owner is invalid") } if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) { @@ -93,7 +94,7 @@ func CheckPerms(path string) error { dacl, _, err := sd.DACL() if err != nil { - return fmt.Errorf("while getting DACL: %w", err) + return errors.Wrap(err, "while getting DACL") } if dacl == nil { @@ -101,7 +102,7 @@ func CheckPerms(path string) error { } if err != nil { - return fmt.Errorf("while looking up current user sid: %w", err) + return errors.Wrap(err, "while looking up current user sid") } rs := reflect.ValueOf(dacl).Elem() @@ -123,7 +124,7 @@ func CheckPerms(path string) error { ace := &AccessAllowedAce{} ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace))) if ret == 0 { - return fmt.Errorf("while getting ACE: %w", windows.GetLastError()) + return errors.Wrap(windows.GetLastError(), "while getting ACE") } log.Debugf("ACE %d: %+v\n", i, ace) @@ -161,14 +162,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT| windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken) if err != nil { - return nil, fmt.Errorf("while opening process token: %w", err) + return nil, errors.Wrapf(err, "while opening process token") } defer procToken.Close() err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation, windows.TokenPrimary, &token) if err != nil { - return nil, fmt.Errorf("while duplicating token: %w", err) + return nil, errors.Wrapf(err, "while duplicating token") } //Remove all privileges from the token @@ -176,7 +177,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil) if err != nil { - return nil, fmt.Errorf("while adjusting token privileges: %w", err) + return nil, errors.Wrapf(err, "while adjusting token privileges") } //Run the plugin as a medium integrity level process @@ -194,7 +195,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { (*byte)(unsafe.Pointer(tml)), tml.Size()) if err != nil { token.Close() - return nil, fmt.Errorf("while setting token information: %w", err) + return nil, errors.Wrapf(err, "while setting token information") } return &windows.SysProcAttr{ @@ -208,7 +209,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { cmd := exec.Command(binaryPath) cmd.SysProcAttr, err = getProcessAtr() if err != nil { - return nil, fmt.Errorf("while getting process attributes: %w", err) + return nil, errors.Wrap(err, "while getting process attributes") } return cmd, err } @@ -228,7 +229,7 @@ func pluginIsValid(path string) error { // check if it exists if _, err = os.Stat(path); err != nil { - return fmt.Errorf("plugin at %s does not exist", path) + return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) } // check if it is owned by root diff --git a/pkg/csplugin/watcher.go b/pkg/csplugin/watcher.go index bec0302e462..983a53c8977 100644 --- a/pkg/csplugin/watcher.go +++ b/pkg/csplugin/watcher.go @@ -4,10 +4,9 @@ import ( "sync" "time" + "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - - "github.com/crowdsecurity/crowdsec/pkg/models" ) /* diff --git a/pkg/csplugin/watcher_test.go b/pkg/csplugin/watcher_test.go index 391a94810f5..94d8d061715 100644 --- a/pkg/csplugin/watcher_test.go +++ b/pkg/csplugin/watcher_test.go @@ -7,12 +7,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - "gopkg.in/tomb.v2" - - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" - "github.com/crowdsecurity/crowdsec/pkg/models" + "gopkg.in/tomb.v2" + "gotest.tools/v3/assert" ) var ctx = context.Background() @@ -67,7 +64,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Microsecond) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") resetTestTomb(&testTomb, &pw) testTomb = tomb.Tomb{} pw.Start(&testTomb) @@ -75,7 +72,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Millisecond*5) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - require.NoError(t, err) + assert.NilError(t, err) resetTestTomb(&testTomb, &pw) // This is to avoid the int complaining } @@ -99,7 +96,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Second) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") // Channel won't contain any events since threshold is not crossed. resetWatcherAlertCounter(&pw) @@ -107,7 +104,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - cstest.RequireErrorContains(t, err, "context deadline exceeded") + assert.ErrorContains(t, err, "context deadline exceeded") // Channel will contain an event since threshold is crossed. resetWatcherAlertCounter(&pw) @@ -115,6 +112,6 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - require.NoError(t, err) + assert.NilError(t, err) resetTestTomb(&testTomb, &pw) } diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 70dea3d7a52..29e6cdf36e2 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -6,13 +6,12 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type Runtime struct { @@ -48,10 +47,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters)) runtime.Cfg = profile if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" { - return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess) + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess) } if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" { - return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) + return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) } for fIdx, filter := range profile.Filters { diff --git a/pkg/csprofiles/csprofiles_test.go b/pkg/csprofiles/csprofiles_test.go index 8adf6829134..81b21e1fb42 100644 --- a/pkg/csprofiles/csprofiles_test.go +++ b/pkg/csprofiles/csprofiles_test.go @@ -5,11 +5,10 @@ import ( "reflect" "testing" - "github.com/stretchr/testify/require" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" + "gotest.tools/v3/assert" ) var ( @@ -96,7 +95,7 @@ func TestNewProfile(t *testing.T) { } profile, _ := NewProfile(profilesCfg) fmt.Printf("expected : %+v | result : %+v", test.expectedNbProfile, len(profile)) - require.Len(t, profile, test.expectedNbProfile) + assert.Equal(t, test.expectedNbProfile, len(profile)) }) } } @@ -200,7 +199,7 @@ func TestEvaluateProfile(t *testing.T) { t.Errorf("EvaluateProfile() got1 = %v, want %v", got1, tt.expectedMatchStatus) } if tt.expectedDuration != "" { - require.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") + assert.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") } }) } diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go index e52922571ef..7bcf814a034 100644 --- a/pkg/cticlient/example/fire.go +++ b/pkg/cticlient/example/fire.go @@ -44,9 +44,6 @@ func main() { } for _, item := range items { - if item.State == "refused" { - continue - } banDuration := time.Until(item.Expiration.Time) allItems = append(allItems, []string{ item.Ip, diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go index 2ad0a6eb34e..1d6550d006a 100644 --- a/pkg/cticlient/types.go +++ b/pkg/cticlient/types.go @@ -120,7 +120,7 @@ type FireItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` - State string `json:"state"` + Status string `json:"status"` Expiration CustomTime `json:"expiration"` } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index b8a09d4c156..0e508d9f1d5 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -148,7 +148,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { finalName := "" f, err := os.Lstat(itemPath) if err != nil { - return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + return nil, errors.Wrapf(err, "while performing lstat on %s", itemPath) } if f.Mode()&os.ModeSymlink == 0 { @@ -158,7 +158,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*resolve the symlink to hub file*/ pathInHub, err := os.Readlink(itemPath) if err != nil { - return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + return nil, errors.Wrapf(err, "while reading symlink of %s", itemPath) } //extract author from path fname := filepath.Base(pathInHub) @@ -240,7 +240,7 @@ func GetInstalledScenariosAsString() ([]string, error) { items, err := GetInstalledScenarios() if err != nil { - return nil, fmt.Errorf("while fetching scenarios: %w", err) + return nil, errors.Wrap(err, "while fetching scenarios") } for _, it := range items { retStr = append(retStr, it.Name) @@ -281,7 +281,7 @@ func GetInstalledParsersAsString() ([]string, error) { items, err := GetInstalledParsers() if err != nil { - return nil, fmt.Errorf("while fetching parsers: %w", err) + return nil, errors.Wrap(err, "while fetching parsers") } for _, it := range items { retStr = append(retStr, it.Name) @@ -308,7 +308,7 @@ func GetInstalledPostOverflowsAsString() ([]string, error) { items, err := GetInstalledPostOverflows() if err != nil { - return nil, fmt.Errorf("while fetching post overflows: %w", err) + return nil, errors.Wrap(err, "while fetching post overflows") } for _, it := range items { retStr = append(retStr, it.Name) @@ -321,9 +321,8 @@ func GetInstalledCollectionsAsString() ([]string, error) { items, err := GetInstalledCollections() if err != nil { - return nil, fmt.Errorf("while fetching collections: %w", err) + return nil, errors.Wrap(err, "while fetching collections") } - for _, it := range items { retStr = append(retStr, it.Name) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index f91b0dcedff..b0e300aa785 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -25,7 +25,7 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -73,7 +73,7 @@ func TestItemStatus(t *testing.T) { } func TestGetters(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -134,7 +134,7 @@ func TestGetters(t *testing.T) { } func TestIndexDownload(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -155,16 +155,10 @@ func getTestCfg() (cfg *csconfig.Config) { return } -func envSetup(t *testing.T) *csconfig.Config { +func envSetup() *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) cfg := getTestCfg() - - defaultTransport := http.DefaultClient.Transport - t.Cleanup(func() { - http.DefaultClient.Transport = defaultTransport - }) - //Mock the http client http.DefaultClient.Transport = newMockTransport() @@ -327,7 +321,7 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) getHubIdxOrFail(t) @@ -359,7 +353,7 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) getHubIdxOrFail(t) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 17b5c0e0022..6923c6d05c3 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -255,7 +256,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { - data := &DataSet{} + data := &types.DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { @@ -271,7 +272,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } if download || force { - err = GetData(data.Data, dataFolder) + err = types.GetData(data.Data, dataFolder) if err != nil { return errors.Wrap(err, "while getting data") } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 4133e227250..af1e938d79d 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -4,12 +4,12 @@ import ( "fmt" "path/filepath" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/enescakir/emoji" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) // pick a hub branch corresponding to the current crowdsec version. @@ -79,11 +79,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err := DownloadLatest(csConfig.Hub, item, force, true) if err != nil { - return fmt.Errorf("while downloading %s: %w", item.Name, err) + return errors.Wrapf(err, "while downloading %s", item.Name) } if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } if downloadOnly { @@ -93,11 +93,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err = EnableItem(csConfig.Hub, item) if err != nil { - return fmt.Errorf("while enabling %s: %w", item.Name, err) + return errors.Wrapf(err, "while enabling %s", item.Name) } if err := AddItem(obtype, item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + return errors.Wrapf(err, "while adding %s", item.Name) } log.Infof("Enabled %s", item.Name) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index bf6e84fb39d..143967002c0 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -9,7 +9,7 @@ import ( //Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection @@ -55,7 +55,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledSceanarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection @@ -103,7 +103,7 @@ func getHubIdxOrFail(t *testing.T) { // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup(t) + cfg := envSetup() defer envTearDown(cfg) // fresh install of collection diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index ecb5d0a3a7c..961e9a02e95 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,17 +2,16 @@ package cwhub import ( "encoding/json" - "errors" "fmt" "os" "path/filepath" "sort" "strings" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) /*the walk/parser_visit function can't receive extra args*/ @@ -369,7 +368,7 @@ func GetHubIdx(hub *csconfig.Hub) error { log.Debugf("loading hub idx %s", hub.HubIndexFile) bidx, err := os.ReadFile(hub.HubIndexFile) if err != nil { - return fmt.Errorf("unable to read index file: %w", err) + return errors.Wrap(err, "unable to read index file") } ret, err := LoadPkgIndex(bidx) if err != nil { diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 688288ceed4..ad011723661 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -9,10 +9,6 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -24,6 +20,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) const ( @@ -202,7 +201,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + return "", errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } decisionDuration, err := time.ParseDuration(*decisionItem.Duration) @@ -392,7 +391,7 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in if rollbackErr != nil { log.Errorf("rollback error: %s", rollbackErr) } - return 0, 0, 0, errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + return 0, 0, 0, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } /*bulk insert some new decisions*/ @@ -565,7 +564,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } marshallMetas, err := json.Marshal(eventItem.Meta) if err != nil { - return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } //the serialized field is too big, let's try to progressively strip it @@ -583,7 +582,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ marshallMetas, err = json.Marshal(eventItem.Meta) if err != nil { - return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } if event.SerializedValidator(string(marshallMetas)) == nil { valid = true @@ -612,7 +611,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert events: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) } } @@ -625,7 +624,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err) } } @@ -638,14 +637,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ duration, err := time.ParseDuration(*decisionItem.Duration) if err != nil { - return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) + return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } /*if the scope is IP or Range, convert the value to integers */ if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) + return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) } } @@ -668,7 +667,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(decisionBulk) == decisionBulkSize { decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) @@ -681,7 +680,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) } @@ -720,7 +719,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(bulk) == bulkSize { alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "bulk creating alert : %s", err) + return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err) } for alertIndex, a := range alerts { ret = append(ret, strconv.Itoa(a.ID)) @@ -729,7 +728,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return nil, fmt.Errorf("error while updating decisions: %s", err) + return []string{}, fmt.Errorf("error while updating decisions: %s", err) } } } @@ -745,7 +744,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return nil, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) + return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) } for alertIndex, a := range alerts { @@ -755,7 +754,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return nil, fmt.Errorf("error while updating decisions: %s", err) + return []string{}, fmt.Errorf("error while updating decisions: %s", err) } } } diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 98bfd45873b..4cd32d83961 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -4,10 +4,9 @@ import ( "fmt" "time" - "github.com/pkg/errors" - "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/bouncer" + "github.com/pkg/errors" ) func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { diff --git a/pkg/database/config.go b/pkg/database/config.go index 8c3578ad596..90a85490f4a 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -1,10 +1,9 @@ package database import ( - "github.com/pkg/errors" - "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/configitem" + "github.com/pkg/errors" ) func (c *Client) GetConfigItem(key string) (*string, error) { diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 5dacc23d4c3..61483f9fd60 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -2,17 +2,17 @@ package database import ( "fmt" - "strconv" "strings" "time" - "entgo.io/ent/dialect/sql" - "github.com/pkg/errors" + "strconv" + "entgo.io/ent/dialect/sql" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" ) type DecisionsByScenario struct { diff --git a/pkg/database/machines.go b/pkg/database/machines.go index 7a010fbfbc4..48243324dcc 100644 --- a/pkg/database/machines.go +++ b/pkg/database/machines.go @@ -5,12 +5,12 @@ import ( "time" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "golang.org/x/crypto/bcrypt" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/machine" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + "golang.org/x/crypto/bcrypt" ) const CapiMachineID = types.CAPIOrigin diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index a616588a76b..a874122ff15 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -175,8 +175,8 @@ func UnmarshalJSON(params ...any) (any, error) { err := json.Unmarshal([]byte(jsonBlob), &out) if err != nil { log.Errorf("UnmarshalJSON : %s", err) - return nil, err + return "", err } target[key] = out - return nil, nil + return "", nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index c1aa4251ca1..36415f746da 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/pkg/errors" ) type HubTest struct { @@ -104,7 +105,7 @@ func (h *HubTest) LoadAllTests() error { for _, f := range testsFolder { if f.IsDir() { if _, err := h.LoadTestItem(f.Name()); err != nil { - return fmt.Errorf("while loading %s: %w", f.Name(), err) + return errors.Wrapf(err, "while loading %s", f.Name()) } } } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 1ec7c5f4480..c3e842bbb2d 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -22,7 +23,7 @@ type HubTestItemConfig struct { LogType string `yaml:"log_type"` Labels map[string]string `yaml:"labels"` IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + OverrideStatics []types.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } type HubIndex struct { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 95400b50d1a..3d52f37e5c3 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -12,14 +12,14 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/fatih/color" + "github.com/pkg/errors" diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type AssertFail struct { @@ -164,7 +164,7 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, fmt.Errorf("while running expression %s: %w", expression, err) + return output, errors.Wrapf(err, "while running expression %s", expression) } return output, nil } diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index 2e2a4e9c8be..d9ec4dddcb7 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -11,11 +11,11 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type ScenarioAssert struct { @@ -149,7 +149,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { env := map[string]interface{}{"results": *s.TestData} if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { - return nil, err + return output, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // log.Warningf("Failed building debugher for %s : %s", assert, err) @@ -162,7 +162,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return nil, fmt.Errorf("while running expression %s: %w", expression, err) + return output, errors.Wrapf(err, "while running expression %s", expression) } return output, nil } diff --git a/pkg/leakybucket/README.md b/pkg/leakybucket/README.md index 4614eddba0a..5254f33b29f 100644 --- a/pkg/leakybucket/README.md +++ b/pkg/leakybucket/README.md @@ -2,102 +2,86 @@ ## Bucket concepts -The Leakybucket is used for decision making. Under certain conditions, -enriched events are poured into these buckets. When these buckets are +Leakybucket is used for decision making. Under certain conditions +enriched events are poured in these buckets. When these buckets are full, we raise a new event. After this event is raised the bucket is destroyed. There are many types of buckets, and we welcome any new useful design of buckets. -Usually, the bucket configuration generates the creation of many -buckets. They are differentiated by a field called stackkey. When two -events arrive with the same stackkey they go in the same matching +Usually the bucket configuration generates the creation of many +buckets. They are differenciated by a field called stackkey. When two +events arrives with the same stackkey they go in the same matching bucket. The very purpose of these buckets is to detect clients that exceed a -certain rate of attempts to do something (ssh connection, http -authentication failure, etc...). Thus, the most used stackkey field is +certain rate of attemps to do something (ssh connection, http +authentication failure, etc...). Thus, the most use stackkey field is often the source_ip. ## Standard leaky buckets Default buckets have two main configuration options: - * capacity: number of events the bucket can hold. When the capacity is reached and a new event is poured, a new event is raised. We call this type of event overflow. This is an int. - * leakspeed: duration needed for an event to leak. When an event - leaks, it disappears from the bucket. + leaks, it disappear from the bucket. ## Trigger -A Trigger is a special type of bucket with a capacity of zero. Thus, when an -event is poured into a trigger, it always raises an overflow. +It's a special type of bucket with a zero capacity. Thus, when an +event is poured in a trigger, it always raises an overflow. ## Uniq -A Uniq is a bucket working like the standard leaky bucket except for one +It's a bucket working as the standard leaky bucket except for one thing: a filter returns a property for each event and only one occurrence of this property is allowed in the bucket, thus the bucket is called uniq. ## Counter -A Counter is a special type of bucket with an infinite capacity and an -infinite leakspeed (it never overflows, nor leaks). Nevertheless, +It's a special type of bucket with an infinite capacity and an +infinite leakspeed (it never overflows, neither leaks). Nevertheless, the event is raised after a fixed duration. The option is called duration. -## Bayesian - -A Bayesian is a special bucket that runs bayesian inference instead of -counting events. Each event must have its likelihoods specified in the -yaml file under `prob_given_benign` and `prob_given_evil`. The bucket -will continue evaluating events until the posterior goes above the -threshold (triggering the overflow) or the duration (specified by leakspeed) -expires. - ## Available configuration options for buckets ### Fields for standard buckets * type: mandatory field. Must be one of "leaky", "trigger", "uniq" or "counter" - -* name: mandatory field, but the value is totally open. Nevertheless, +* name: mandatory field, but the value is totally open. Nevertheless this value will tag the events raised by the bucket. - -* filter: mandatory field. It's a filter that is run to decide whether - an event matches the bucket or not. The filter has to return +* filter: mandatory field. It's a filter that is run when the decision + to make an event match the bucket or not. The filter have to return a boolean. As a filter implementation we use https://github.com/antonmedv/expr - * capacity: [mandatory for now, shouldn't be mandatory in the final version] it's the size of the bucket. When pouring in a bucket already with size events, it overflows. - -* leakspeed: leakspeed is a time duration (it has to be parsed by - https://golang.org/pkg/time/#ParseDuration). After each interval, an +* leakspeed: leakspeed is a time duration (has to be parseable by + https://golang.org/pkg/time/#ParseDuration). After each interval an event is leaked from the bucket. - * stackkey: mandatory field. This field is used to differentiate on - which instance of the bucket the matching events will be poured. - When an unknown stackkey is seen in an event, a new bucket is created. - -* on_overflow: optional field, that tells what to do when the - bucket is returning the overflow event. As of today, the possibilities - are "ban,1h", "Reprocess" or "Delete". - Reprocess is used to send the raised event back to the event pool to - be matched against buckets + which bucket ongoing events will be poured. When an unknown stackkey + is seen in an event a new bucket is created. +* on_overflow: optional field, that tells the what to do when the + bucket is returning the overflow event. As of today, the possibility + are these: "ban,1h", "Reprocess", "Delete". + Reprocess is used to send the raised event back in the event pool to + be matched agains buckets ### Fields for special buckets #### Uniq - * uniq_filter: an expression that must comply with the syntax defined - in https://github.com/antonmedv/expr and must return a string. - All strings returned by this filter in the same buckets have to be different. - Thus if a string is seen twice, the event is dismissed. +Uniq has an extra field uniq_filter which is too use the filter +implementation from https://github.com/antonmedv/expr. The filter must +return a string. All strins returned by this filter in the same +buckets have to be different. Thus, if a string is seen twice it is +dismissed. #### Trigger @@ -105,27 +89,11 @@ Capacity and leakspeed are not relevant for this kind of bucket. #### Counter - * duration: the Counter will be destroyed after this interval - has elapsed since its creation. The duration must be parsed - by https://golang.org/pkg/time/#ParseDuration. - Nevertheless, this kind of bucket is often used with an infinite - leakspeed and an infinite capacity [capacity set to -1 for now]. - -#### Bayesian - - * bayesian_prior: The prior to start with - * bayesian_threshold: The threshold for the posterior to trigger the overflow. - * bayesian_conditions: List of Bayesian conditions with likelihoods - -Bayesian Conditions are built from: - * condition: The expr for this specific condition to be true - * prob_given_evil: The likelihood an IP satisfies the condition given the fact - that it is a maliscious IP - * prob_given_benign: The likelihood an IP satisfies the condition given the fact - that it is a benign IP - * guillotine: Bool to stop the condition from getting evaluated if it has - evaluated to true once. This should be used if evaluating the condition is - computationally expensive. +It's a special kind of bucket that raise an event and is destroyed +after a fixed duration. The configuration field used is duration and +must be parseable by https://golang.org/pkg/time/#ParseDuration. +Nevertheless, this kind of bucket is often used with an infinite +leakspeed and an infinite capacity [capacity set to -1 for now]. ## Add examples here @@ -158,17 +126,17 @@ Bayesian Conditions are built from: [This is not dry enough to have many details here, but:] -The bucket code is triggered by runPour in pour.go, by calling the `leaky.PourItemToHolders` function. -There is one struct called buckets which is for now a +The bucket code is triggered by `InfiniBucketify` in main.go. +There's one struct called buckets which is for now a `map[string]interface{}` that holds all buckets. The key of this map -is derived from the filter configured for the bucket and its -stackkey. This looks complicated, but it allows us to use -only one struct. This is done in buckets.go. +is derivated from the filter configured for the bucket and its +stackkey. This looks like complicated, but in fact it allows us to use +only one structs. This is done in buckets.go. -On top of that the implementation defines only the standard leaky -bucket. A goroutine is launched for every bucket (`bucket.go`). This +On top of that the implementation define only the standard leaky +bucket. A goroutine is launched for every buckets (bucket.go). This goroutine manages the life of the bucket. For special buckets, hooks are defined at initialization time in -manager.go. Hooks are called when relevant by the bucket goroutine -when events are poured and/or when a bucket overflows. +manager.go. Hooks are called when relevant by the bucket gorourine +when events are poured and/or when bucket overflows. \ No newline at end of file diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go deleted file mode 100644 index bd9aaed96b4..00000000000 --- a/pkg/leakybucket/bayesian.go +++ /dev/null @@ -1,163 +0,0 @@ -package leakybucket - -import ( - "fmt" - - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" -) - -type RawBayesianCondition struct { - ConditionalFilterName string `yaml:"condition"` - ProbGivenEvil float32 `yaml:"prob_given_evil"` - ProbGivenBenign float32 `yaml:"prob_given_benign"` - Guillotine bool `yaml:"guillotine,omitempty"` -} - -type BayesianEvent struct { - rawCondition RawBayesianCondition - conditionalFilterRuntime *vm.Program - guillotineState bool -} - -type BayesianBucket struct { - bayesianEventArray []*BayesianEvent - prior float32 - threshold float32 - posterior float32 - DumbProcessor -} - -func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { - numerator := probGivenEvil * prior - denominator := numerator + ProbGivenBenign*(1-prior) - - return numerator / denominator -} - -func (c *BayesianBucket) OnBucketInit(g *BucketFactory) error { - var err error - BayesianEventArray := make([]*BayesianEvent, len(g.BayesianConditions)) - - if conditionalExprCache == nil { - conditionalExprCache = make(map[string]vm.Program) - } - conditionalExprCacheLock.Lock() - - for index, bcond := range g.BayesianConditions { - var bayesianEvent BayesianEvent - bayesianEvent.rawCondition = bcond - err = bayesianEvent.compileCondition() - if err != nil { - return err - } - BayesianEventArray[index] = &bayesianEvent - } - conditionalExprCacheLock.Unlock() - c.bayesianEventArray = BayesianEventArray - - c.prior = g.BayesianPrior - c.threshold = g.BayesianThreshold - - return err -} - -func (c *BayesianBucket) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event { - return func(msg types.Event, l *Leaky) *types.Event { - c.posterior = c.prior - l.logger.Debugf("starting bayesian evaluation with prior: %v", c.posterior) - - for _, bevent := range c.bayesianEventArray { - err := bevent.bayesianUpdate(c, msg, l) - if err != nil { - l.logger.Errorf("bayesian update failed for %s with %s", bevent.rawCondition.ConditionalFilterName, err) - } - } - - l.logger.Debugf("value of posterior after events : %v", c.posterior) - - if c.posterior > c.threshold { - l.logger.Debugf("Bayesian bucket overflow") - l.Ovflw_ts = l.Last_ts - l.Out <- l.Queue - return nil - } - - return &msg - } -} - -func (b *BayesianEvent) bayesianUpdate(c *BayesianBucket, msg types.Event, l *Leaky) error { - var condition, ok bool - - if b.conditionalFilterRuntime == nil { - l.logger.Tracef("empty conditional filter runtime for %s", b.rawCondition.ConditionalFilterName) - return nil - } - - l.logger.Tracef("guillotine value for %s : %v", b.rawCondition.ConditionalFilterName, b.getGuillotineState()) - if b.getGuillotineState() { - l.logger.Tracef("guillotine already triggered for %s", b.rawCondition.ConditionalFilterName) - l.logger.Tracef("condition true updating prior for: %s", b.rawCondition.ConditionalFilterName) - c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) - l.logger.Tracef("new value of posterior : %v", c.posterior) - return nil - } - - l.logger.Debugf("running condition expression: %s", b.rawCondition.ConditionalFilterName) - ret, err := expr.Run(b.conditionalFilterRuntime, map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}) - if err != nil { - return fmt.Errorf("unable to run conditional filter: %s", err) - } - - l.logger.Tracef("bayesian bucket expression %s returned : %v", b.rawCondition.ConditionalFilterName, ret) - if condition, ok = ret.(bool); !ok { - return fmt.Errorf("bayesian condition unexpected non-bool return: %T", ret) - } - - l.logger.Tracef("condition %T updating prior for: %s", condition, b.rawCondition.ConditionalFilterName) - if condition { - c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) - b.triggerGuillotine() - } else { - c.posterior = updateProbability(c.posterior, 1-b.rawCondition.ProbGivenEvil, 1-b.rawCondition.ProbGivenBenign) - } - l.logger.Tracef("new value of posterior: %v", c.posterior) - - return nil -} - -func (b *BayesianEvent) getGuillotineState() bool { - if b.rawCondition.Guillotine { - return b.guillotineState - } - return false -} - -func (b *BayesianEvent) triggerGuillotine() { - b.guillotineState = true -} - -func (b *BayesianEvent) compileCondition() error { - var err error - var compiledExpr *vm.Program - - if compiled, ok := conditionalExprCache[b.rawCondition.ConditionalFilterName]; ok { - b.conditionalFilterRuntime = &compiled - return nil - } - - conditionalExprCacheLock.Unlock() - //release the lock during compile same as coditional bucket - compiledExpr, err = expr.Compile(b.rawCondition.ConditionalFilterName, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})...) - if err != nil { - return fmt.Errorf("bayesian condition compile error: %w", err) - } - b.conditionalFilterRuntime = compiledExpr - conditionalExprCacheLock.Lock() - conditionalExprCache[b.rawCondition.ConditionalFilterName] = *compiledExpr - - return nil -} diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index 286c51f113c..004d5b9d884 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -191,10 +191,6 @@ func FromFactory(bucketFactory BucketFactory) *Leaky { l.conditionalOverflow = true l.Duration = l.BucketConfig.leakspeed } - - if l.BucketConfig.Type == "bayesian" { - l.Duration = l.BucketConfig.leakspeed - } return l } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index dc1f4ed51d8..1e212f815fe 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -51,9 +51,6 @@ type BucketFactory struct { Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc. OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow - BayesianPrior float32 `yaml:"bayesian_prior"` - BayesianThreshold float32 `yaml:"bayesian_threshold"` - BayesianConditions []RawBayesianCondition `yaml:"bayesian_conditions"` //conditions for the bayesian bucket ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP BucketName string `yaml:"-"` Filename string `yaml:"-"` @@ -123,25 +120,6 @@ func ValidateFactory(bucketFactory *BucketFactory) error { if bucketFactory.leakspeed == 0 { return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed) } - } else if bucketFactory.Type == "bayesian" { - if bucketFactory.BayesianConditions == nil { - return fmt.Errorf("bayesian bucket must have bayesian conditions") - } - if bucketFactory.BayesianPrior == 0 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") - } - if bucketFactory.BayesianThreshold == 0 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") - } - if bucketFactory.BayesianPrior > 1 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") - } - if bucketFactory.BayesianThreshold > 1 { - return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") - } - if bucketFactory.Capacity != -1 { - return fmt.Errorf("bayesian bucket must have capacity -1") - } } else { return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type) } @@ -338,8 +316,6 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) case "conditional": bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) - case "bayesian": - bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) default: return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err) } @@ -379,11 +355,6 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &ConditionalOverflow{}) } - if bucketFactory.BayesianThreshold != 0 { - bucketFactory.logger.Tracef("Adding bayesian processor") - bucketFactory.processors = append(bucketFactory.processors, &BayesianBucket{}) - } - if len(bucketFactory.Data) > 0 { for _, data := range bucketFactory.Data { if data.DestPath == "" { diff --git a/pkg/leakybucket/manager_load_test.go b/pkg/leakybucket/manager_load_test.go index 513f11ff373..bb3df75cd92 100644 --- a/pkg/leakybucket/manager_load_test.go +++ b/pkg/leakybucket/manager_load_test.go @@ -119,25 +119,3 @@ func TestCounterBucketsConfig(t *testing.T) { } } - -func TestBayesianBucketsConfig(t *testing.T) { - var CfgTests = []cfgTest{ - - //basic valid counter - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, true, true}, - //bad capacity - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: 1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //missing prior - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //missing threshold - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //bad prior - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 1.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - //bad threshold - {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 1.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, - } - if err := runTest(CfgTests); err != nil { - t.Fatalf("%s", err) - } - -} diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index 45446b8f65e..d6131cd26b9 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -6,14 +6,15 @@ import ( "sort" "strconv" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/alertcontext" - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/antonmedv/expr" ) // SourceFromEvent extracts and formats a valid models.Source object from an Event @@ -52,7 +53,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) if !ok { @@ -127,7 +128,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) @@ -144,7 +145,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e } retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, fmt.Errorf("while running scope filter: %w", err) + return srcs, errors.Wrapf(err, "while running scope filter") } value, ok := retValue.(string) @@ -216,7 +217,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st for _, evt := range queue.Queue { srcs, err := SourceFromEvent(evt, leaky) if err != nil { - return nil, "", fmt.Errorf("while extracting scope from bucket %s: %w", leaky.Name, err) + return nil, "", errors.Wrapf(err, "while extracting scope from bucket %s", leaky.Name) } for key, src := range srcs { if source_type == types.Undefined { @@ -275,7 +276,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { //Get the sources from Leaky/Queue sources, source_scope, err := alertFormatSource(leaky, queue) if err != nil { - return runtimeAlert, fmt.Errorf("unable to collect sources from bucket: %w", err) + return runtimeAlert, errors.Wrap(err, "unable to collect sources from bucket") } runtimeAlert.Sources = sources //Include source info in format string diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml deleted file mode 100644 index 8e8c26e6f8d..00000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,21 +0,0 @@ -type: bayesian -name: test/guillotine-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: evt.Meta.http_path == "/" - prob_given_evil: 0.8 - prob_given_benign: 0.2 - guillotine : true -- condition: evt.Meta.ssh_user == "admin" - prob_given_evil: 0.9 - prob_given_benign: 0.5 - guillotine : true -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf91..00000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json deleted file mode 100644 index 07b7b6a6ee5..00000000000 --- a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/guillotine-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml deleted file mode 100644 index 1110fb7b82d..00000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,21 +0,0 @@ -type: bayesian -name: test/multiple-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: evt.Meta.http_path == "/" - prob_given_evil: 0.8 - prob_given_benign: 0.2 - guillotine : true -- condition: evt.Meta.ssh_user == "admin" - prob_given_evil: 0.9 - prob_given_benign: 0.5 - guillotine : true -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf91..00000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json deleted file mode 100644 index 69454a6edf9..00000000000 --- a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "1.2.3.4", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/multiple-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml deleted file mode 100644 index 21a4ab07495..00000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml +++ /dev/null @@ -1,19 +0,0 @@ -type: bayesian -name: test/simple-bayesian -debug: true -description: "bayesian bucket" -filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" -groupby: evt.Meta.source_ip -bayesian_prior: 0.5 -bayesian_threshold: 0.8 -bayesian_conditions: -- condition: any(queue.Queue, {.Meta.http_path == "/"}) - prob_given_evil: 0.8 - prob_given_benign: 0.2 -- condition: any(queue.Queue, {.Meta.ssh_user == "admin"}) - prob_given_evil: 0.9 - prob_given_benign: 0.5 -leakspeed: 30s -capacity: -1 -labels: - type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml deleted file mode 100644 index 05e1557cf91..00000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml +++ /dev/null @@ -1 +0,0 @@ - - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json deleted file mode 100644 index a5807c4b6f0..00000000000 --- a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "lines": [ - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "http_access-log", - "http_path": "/" - } - }, - { - "Line": { - "Labels": { - "type": "nginx" - }, - "Raw": "don't care" - }, - "MarshaledTime": "2020-01-01T10:00:00.000Z", - "Meta": { - "source_ip": "2a00:1450:4007:816::200e", - "log_type": "ssh_access-log", - "ssh_user": "admin" - } - } - ], - "results": [ - { - "Type" : 1, - "Alert": { - "sources" : { - "2a00:1450:4007:816::200e": { - "ip": "2a00:1450:4007:816::200e", - "scope": "Ip", - "value": "2a00:1450:4007:816::200e" - } - }, - "Alert" : { - "scenario": "test/simple-bayesian", - "events_count": 2 - } - } - } - ] - } \ No newline at end of file diff --git a/pkg/metabase/api.go b/pkg/metabase/api.go index 7235ff7f104..99cbf9ec77f 100644 --- a/pkg/metabase/api.go +++ b/pkg/metabase/api.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" ) -type MBClient struct { +type APIClient struct { CTX *sling.Sling Client *http.Client } @@ -35,15 +35,15 @@ var ( } ) -func NewMBClient(url string) (*MBClient, error) { +func NewAPIClient(url string) (*APIClient, error) { httpClient := &http.Client{Timeout: 20 * time.Second} - return &MBClient{ + return &APIClient{ CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())), Client: httpClient, }, nil } -func (h *MBClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { +func (h *APIClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { var Success interface{} var Error interface{} var resp *http.Response @@ -80,6 +80,6 @@ func (h *MBClient) Do(method string, route string, body interface{}) (interface{ } // Set set headers as key:value -func (h *MBClient) Set(key string, value string) { +func (h *APIClient) Set(key string, value string) { h.CTX = h.CTX.Set(key, value) } diff --git a/pkg/metabase/database.go b/pkg/metabase/database.go index 273d06dee16..0a7890faceb 100644 --- a/pkg/metabase/database.go +++ b/pkg/metabase/database.go @@ -7,13 +7,14 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" ) type Database struct { DBUrl string Model *Model Config *csconfig.DatabaseCfg - Client *MBClient + Client *APIClient Details *Details // in case mysql host is 127.0.0.1 the ip address of mysql/pgsql host will be the docker gateway since metabase run in a container } @@ -40,7 +41,7 @@ type Model struct { Schedules map[string]interface{} `json:"schedules"` } -func NewDatabase(config *csconfig.DatabaseCfg, client *MBClient, remoteDBAddr string) (*Database, error) { +func NewDatabase(config *csconfig.DatabaseCfg, client *APIClient, remoteDBAddr string) (*Database, error) { var details *Details database := Database{} @@ -79,13 +80,13 @@ func (d *Database) Update() error { data, err := json.Marshal(success) if err != nil { - return fmt.Errorf("update sqlite db response (marshal): %w", err) + return errors.Wrap(err, "update sqlite db response (marshal)") } model := Model{} if err := json.Unmarshal(data, &model); err != nil { - return fmt.Errorf("update sqlite db response (unmarshal): %w", err) + return errors.Wrap(err, "update sqlite db response (unmarshal)") } model.Details = d.Details _, errormsg, err = d.Client.Do("PUT", routes[databaseEndpoint], model) diff --git a/pkg/metabase/metabase.go b/pkg/metabase/metabase.go index cdbe65ec8df..c12d9166f81 100644 --- a/pkg/metabase/metabase.go +++ b/pkg/metabase/metabase.go @@ -4,7 +4,6 @@ import ( "archive/zip" "bytes" "context" - "errors" "fmt" "io" "net/http" @@ -16,14 +15,15 @@ import ( "github.com/docker/docker/client" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) type Metabase struct { Config *Config - Client *MBClient + Client *APIClient Container *Container Database *Database InternalDBURL string @@ -80,7 +80,7 @@ func (m *Metabase) Init(containerName string) error { return fmt.Errorf("database '%s' not supported", m.Config.Database.Type) } - m.Client, err = NewMBClient(m.Config.ListenURL) + m.Client, err = NewAPIClient(m.Config.ListenURL) if err != nil { return err } @@ -90,7 +90,7 @@ func (m *Metabase) Init(containerName string) error { } m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, metabaseImage, DBConnectionURI, m.Config.DockerGroupID) if err != nil { - return fmt.Errorf("container init: %w", err) + return errors.Wrap(err, "container init") } return nil @@ -151,36 +151,36 @@ func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort }, } if err := metabase.Init(containerName); err != nil { - return nil, fmt.Errorf("metabase setup init: %w", err) + return nil, errors.Wrap(err, "metabase setup init") } if err := metabase.DownloadDatabase(false); err != nil { - return nil, fmt.Errorf("metabase db download: %w", err) + return nil, errors.Wrap(err, "metabase db download") } if err := metabase.Container.Create(); err != nil { - return nil, fmt.Errorf("container create: %w", err) + return nil, errors.Wrap(err, "container create") } if err := metabase.Container.Start(); err != nil { - return nil, fmt.Errorf("container start: %w", err) + return nil, errors.Wrap(err, "container start") } log.Infof("waiting for metabase to be up (can take up to a minute)") if err := metabase.WaitAlive(); err != nil { - return nil, fmt.Errorf("wait alive: %w", err) + return nil, errors.Wrap(err, "wait alive") } if err := metabase.Database.Update(); err != nil { - return nil, fmt.Errorf("update database: %w", err) + return nil, errors.Wrap(err, "update database") } if err := metabase.Scan(); err != nil { - return nil, fmt.Errorf("db scan: %w", err) + return nil, errors.Wrap(err, "db scan") } if err := metabase.ResetCredentials(); err != nil { - return nil, fmt.Errorf("reset creds: %w", err) + return nil, errors.Wrap(err, "reset creds") } return metabase, nil @@ -193,7 +193,7 @@ func (m *Metabase) WaitAlive() error { if err != nil { if strings.Contains(err.Error(), "password:did not match stored password") { log.Errorf("Password mismatch error, is your dashboard already setup ? Run 'cscli dashboard remove' to reset it.") - return fmt.Errorf("password mismatch error: %w", err) + return errors.Wrapf(err, "Password mismatch error") } log.Debugf("%+v", err) } else { @@ -215,7 +215,7 @@ func (m *Metabase) Login(username string, password string) error { } if errormsg != nil { - return fmt.Errorf("http login: %s", errormsg) + return errors.Wrap(err, "http login") } resp, ok := successmsg.(map[string]interface{}) if !ok { @@ -238,7 +238,7 @@ func (m *Metabase) Scan() error { return err } if errormsg != nil { - return fmt.Errorf("http scan: %s", errormsg) + return errors.Wrap(err, "http scan") } return nil @@ -252,10 +252,10 @@ func (m *Metabase) ResetPassword(current string, newPassword string) error { } _, errormsg, err := m.Client.Do("PUT", routes[resetPasswordEndpoint], body) if err != nil { - return fmt.Errorf("reset username: %w", err) + return errors.Wrap(err, "reset username") } if errormsg != nil { - return fmt.Errorf("http reset password: %s", errormsg) + return errors.Wrap(err, "http reset password") } return nil } @@ -275,11 +275,11 @@ func (m *Metabase) ResetUsername(username string) error { _, errormsg, err := m.Client.Do("PUT", routes[userEndpoint], body) if err != nil { - return fmt.Errorf("reset username: %w", err) + return errors.Wrap(err, "reset username") } if errormsg != nil { - return fmt.Errorf("http reset username: %s", errormsg) + return errors.Wrap(err, "http reset username") } return nil diff --git a/pkg/parser/enrich.go b/pkg/parser/enrich.go index 5180b9a5fb9..04652407dd6 100644 --- a/pkg/parser/enrich.go +++ b/pkg/parser/enrich.go @@ -1,12 +1,11 @@ package parser import ( - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) -/* should be part of a package shared with enrich/geoip.go */ +/* should be part of a packaged shared with enrich/geoip.go */ type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type InitFunc func(map[string]string) (interface{}, error) diff --git a/pkg/parser/enrich_date.go b/pkg/parser/enrich_date.go index 20828af9037..a1dd994be97 100644 --- a/pkg/parser/enrich_date.go +++ b/pkg/parser/enrich_date.go @@ -3,10 +3,9 @@ package parser import ( "time" - log "github.com/sirupsen/logrus" - expr "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) func parseDateWithFormat(date, format string) (string, time.Time) { diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go index f622e6c359a..5bbc0e94df6 100644 --- a/pkg/parser/enrich_dns.go +++ b/pkg/parser/enrich_dns.go @@ -3,9 +3,9 @@ package parser import ( "net" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + //"github.com/crowdsecurity/crowdsec/pkg/parser" ) /* All plugins must export a list of function pointers for exported symbols */ diff --git a/pkg/parser/enrich_geoip.go b/pkg/parser/enrich_geoip.go index 0a263c82793..8c25bef4464 100644 --- a/pkg/parser/enrich_geoip.go +++ b/pkg/parser/enrich_geoip.go @@ -5,11 +5,11 @@ import ( "net" "strconv" - "github.com/oschwald/geoip2-golang" - "github.com/oschwald/maxminddb-golang" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/oschwald/geoip2-golang" + "github.com/oschwald/maxminddb-golang" ) func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/enrich_unmarshal.go b/pkg/parser/enrich_unmarshal.go index dce9c75d466..404340401f6 100644 --- a/pkg/parser/enrich_unmarshal.go +++ b/pkg/parser/enrich_unmarshal.go @@ -3,9 +3,8 @@ package parser import ( "encoding/json" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" ) func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 036a244cca8..2370d37960c 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -1,24 +1,24 @@ package parser import ( - "errors" "fmt" "net" "strings" "time" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/davecgh/go-spew/spew" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" - yaml "gopkg.in/yaml.v2" - "github.com/crowdsecurity/grokky" + "github.com/pkg/errors" + yaml "gopkg.in/yaml.v2" + "github.com/antonmedv/expr/vm" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" ) type Node struct { @@ -56,11 +56,11 @@ type Node struct { SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"` //Holds a grok pattern - Grok GrokPattern `yaml:"grok,omitempty"` + Grok types.GrokPattern `yaml:"grok,omitempty"` //Statics can be present in any type of node and is executed last - Statics []ExtraField `yaml:"statics,omitempty"` + Statics []types.ExtraField `yaml:"statics,omitempty"` //Stash allows to capture data from the log line and store it in an accessible cache - Stash []DataCapture `yaml:"stash,omitempty"` + Stash []types.DataCapture `yaml:"stash,omitempty"` //Whitelists Whitelist Whitelist `yaml:"whitelist,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"` @@ -360,27 +360,29 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } //Iterate on leafs - for _, leaf := range n.LeavesNodes { - ret, err := leaf.process(p, ctx, cachedExprEnv) - if err != nil { - clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) - clog.Debugf("Event leaving node : ko") - return false, err - } - clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) - if ret { - NodeState = true - /* if child is successful, stop processing */ - if n.OnSuccess == "next_stage" { - clog.Debugf("child is success, OnSuccess=next_stage, skip") - break - } - } else if !NodeHasOKGrok { - /* - If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. - If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. - */ - NodeState = false + if len(n.LeavesNodes) > 0 { + for _, leaf := range n.LeavesNodes { + ret, err := leaf.process(p, ctx, cachedExprEnv) + if err != nil { + clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) + clog.Debugf("Event leaving node : ko") + return false, err + } + clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) + if ret { + NodeState = true + /* if child is successful, stop processing */ + if n.OnSuccess == "next_stage" { + clog.Debugf("child is success, OnSuccess=next_stage, skip") + break + } + } else if !NodeHasOKGrok { + /* + If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. + If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. + */ + NodeState = false + } } } /*todo : check if a node made the state change ?*/ @@ -400,11 +402,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri if n.Name != "" { NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() } - /* - This is to apply statics when the node *has* whitelists that successfully matched the node. + Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node. */ - if len(n.Statics) > 0 && (isWhitelisted || !hasWhitelist) { + if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist { clog.Debugf("+ Processing %d statics", len(n.Statics)) // if all else is good in whitelist, process node's statics err := n.ProcessStatics(n.Statics, p) @@ -452,8 +453,8 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* if the node has debugging enabled, create a specific logger with debug that will be used only for processing this node ;) */ if n.Debug { - var clog = log.New() - if err = types.ConfigureLogger(clog); err != nil { + var clog = logrus.New() + if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) @@ -492,7 +493,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* handle pattern_syntax and groks */ for _, pattern := range n.SubGroks { n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value) - if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { + if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { if errors.Is(err, grokky.ErrAlreadyExist) { n.Logger.Warningf("grok '%s' already registred", pattern.Key) continue @@ -535,18 +536,20 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling grok's expression: %w", err) + return errors.Wrap(err, "while compiling grok's expression") } } /* load grok statics */ - //compile expr statics if present - for idx := range n.Grok.Statics { - if n.Grok.Statics[idx].ExpValue != "" { - n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, - exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) - if err != nil { - return err + if len(n.Grok.Statics) > 0 { + //compile expr statics if present + for idx := range n.Grok.Statics { + if n.Grok.Statics[idx].ExpValue != "" { + n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, + exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + if err != nil { + return err + } } } valid = true @@ -557,53 +560,54 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Stash[i].ValueExpression, err = expr.Compile(stash.Value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling stash value expression: %w", err) + return errors.Wrap(err, "while compiling stash value expression") } n.Stash[i].KeyExpression, err = expr.Compile(stash.Key, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return fmt.Errorf("while compiling stash key expression: %w", err) + return errors.Wrap(err, "while compiling stash key expression") } n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL) if err != nil { - return fmt.Errorf("while parsing stash ttl: %w", err) + return errors.Wrap(err, "while parsing stash ttl") } logLvl := n.Logger.Logger.GetLevel() //init the cache, does it make sense to create it here just to be sure everything is fine ? - if err = cache.CacheInit(cache.CacheCfg{ + if err := cache.CacheInit(cache.CacheCfg{ Size: n.Stash[i].MaxMapSize, TTL: n.Stash[i].TTLVal, Name: n.Stash[i].Name, Strategy: n.Stash[i].Strategy, LogLevel: &logLvl, }); err != nil { - return fmt.Errorf("while initializing cache: %w", err) + return errors.Wrap(err, "while initializing cache") } } /* compile leafs if present */ - for idx := range n.LeavesNodes { - if n.LeavesNodes[idx].Name == "" { - n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) - } - /*propagate debug/stats to child nodes*/ - if !n.LeavesNodes[idx].Debug && n.Debug { - n.LeavesNodes[idx].Debug = true - } - if !n.LeavesNodes[idx].Profiling && n.Profiling { - n.LeavesNodes[idx].Profiling = true - } - n.LeavesNodes[idx].Stage = n.Stage - err = n.LeavesNodes[idx].compile(pctx, ectx) - if err != nil { - return err + if len(n.LeavesNodes) > 0 { + for idx := range n.LeavesNodes { + if n.LeavesNodes[idx].Name == "" { + n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } + /*propagate debug/stats to child nodes*/ + if !n.LeavesNodes[idx].Debug && n.Debug { + n.LeavesNodes[idx].Debug = true + } + if !n.LeavesNodes[idx].Profiling && n.Profiling { + n.LeavesNodes[idx].Profiling = true + } + n.LeavesNodes[idx].Stage = n.Stage + err = n.LeavesNodes[idx].compile(pctx, ectx) + if err != nil { + return err + } } valid = true } - /* load statics if present */ for idx := range n.Statics { if n.Statics[idx].ExpValue != "" { @@ -622,7 +626,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding ip %s to whitelists", net.ParseIP(v)) valid = true } - for _, v := range n.Whitelist.Cidrs { _, tnet, err := net.ParseCIDR(v) if err != nil { @@ -632,7 +635,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding cidr %s to whitelists", tnet) valid = true } - for _, filter := range n.Whitelist.Exprs { expression := &ExprWhitelist{} expression.Filter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) @@ -658,6 +660,5 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { if err := n.validate(pctx, ectx); err != nil { return err } - return nil } diff --git a/pkg/parser/node_test.go b/pkg/parser/node_test.go index d85aa82a8ae..f529cb4d40c 100644 --- a/pkg/parser/node_test.go +++ b/pkg/parser/node_test.go @@ -3,6 +3,7 @@ package parser import ( "testing" + "github.com/crowdsecurity/crowdsec/pkg/types" yaml "gopkg.in/yaml.v2" ) @@ -19,7 +20,7 @@ func TestParserConfigs(t *testing.T) { Valid bool }{ //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //bad filter {&Node{Debug: true, Stage: "s00", Filter: "ratata"}, false, false}, //empty node @@ -27,25 +28,25 @@ func TestParserConfigs(t *testing.T) { //bad subgrok {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true}, //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, //bad node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, //ok node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //valid node with grok sub-pattern used by name - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, //node with unexisting grok pattern - {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, + {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, //node with grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROK"), Value: string("[a-z]")}, {Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")}, - }, Grok: GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, + }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, //node with broken grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")}, {Key: string("MYGROKBIS"), Value: string("[a-z]")}, - }, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, + }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, } for idx := range CfgTests { err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{}) @@ -63,5 +64,6 @@ func TestParserConfigs(t *testing.T) { if CfgTests[idx].Valid == false && err == nil { t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests)) } + } } diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go index 04d08cc2785..089b95835b5 100644 --- a/pkg/parser/parsing_test.go +++ b/pkg/parser/parsing_test.go @@ -11,12 +11,11 @@ import ( "strings" "testing" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" ) type TestFile struct { @@ -55,6 +54,7 @@ func TestParser(t *testing.T) { } } } + } func BenchmarkParser(t *testing.B) { @@ -90,6 +90,7 @@ func BenchmarkParser(t *testing.B) { } func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error { + var ( err error pnodes []Node @@ -111,7 +112,7 @@ func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing if err != nil { panic(err) } - if err = yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { + if err := yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { return fmt.Errorf("failed unmarshaling %s : %s", parser_cfg_file, err) } @@ -398,7 +399,7 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } for _, k := range p { - if _, err := fmt.Fprintf(f, "## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value); err != nil { + if _, err := f.WriteString(fmt.Sprintf("## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value)); err != nil { t.Fatal("failed to write to file") } fmt.Printf("%v\t%v\n", k.Key, k.Value) @@ -413,4 +414,5 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } f.Close() + } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index e6ee07ea765..3c7d382d577 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -9,21 +9,24 @@ import ( "errors" "fmt" "reflect" - "strconv" "strings" "sync" "time" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/types" + + "strconv" + "github.com/mohae/deepcopy" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/antonmedv/expr" ) /* ok, this is kinda experimental, I don't know how bad of an idea it is .. */ func SetTargetByName(target string, value string, evt *types.Event) bool { + if evt == nil { return false } @@ -68,6 +71,8 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { tmp = reflect.Indirect(tmp) } iter = tmp + //nolint: gosimple + break case reflect.Ptr: tmp := iter.Elem() iter = reflect.Indirect(tmp.FieldByName(f)) @@ -89,24 +94,24 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { return true } -func printStaticTarget(static ExtraField) string { - switch { - case static.Method != "": +func printStaticTarget(static types.ExtraField) string { + + if static.Method != "" { return static.Method - case static.Parsed != "": + } else if static.Parsed != "" { return fmt.Sprintf(".Parsed[%s]", static.Parsed) - case static.Meta != "": + } else if static.Meta != "" { return fmt.Sprintf(".Meta[%s]", static.Meta) - case static.Enriched != "": + } else if static.Enriched != "" { return fmt.Sprintf(".Enriched[%s]", static.Enriched) - case static.TargetByName != "": + } else if static.TargetByName != "" { return static.TargetByName - default: + } else { return "?" } } -func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { +func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) error { //we have a few cases : //(meta||key) + (static||reference||expr) var value string @@ -192,6 +197,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { } else { clog.Fatal("unable to process static : unknown target") } + } return nil } @@ -351,8 +357,10 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) event.Process = false return event, nil } + } event.Process = true return event, nil + } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 37d43fbfe96..3bdaed1f0e9 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -109,16 +109,17 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( continue } - for _, data := range node.Data { - err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) - if err != nil { - log.Error(err) - } - if data.Type == "regexp" { //cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) + if len(node.Data) > 0 { + for _, data := range node.Data { + err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) + if err != nil { + log.Error(err) + } + if data.Type == "regexp" { //cache only makes sense for regexp + exprhelpers.RegexpCacheInit(data.DestPath, *data) + } } } - nodes = append(nodes, node) nodesCount++ } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index d5d91f9320d..b0f38f81e6c 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -7,13 +7,12 @@ import ( "sort" "strings" - log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/grokky" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/fflag" + + "github.com/crowdsecurity/grokky" + log "github.com/sirupsen/logrus" ) type UnixParserCtx struct { @@ -118,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) if err != nil { - return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) + return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err) } /* @@ -136,8 +135,8 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { log.Infof("Loading postoverflow parsers") parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx) } else { - log.Infof("No postoverflow parsers to load") parsers.Povfwnodes = []Node{} + log.Infof("No postoverflow parsers to load") } if err != nil { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index e2f179fb3a1..c56ad40a728 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -4,7 +4,6 @@ import ( "net" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index b345c0d6f63..957f70a258d 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -7,12 +7,15 @@ import ( "os/exec" "sort" - "github.com/Masterminds/semver/v3" + "github.com/Masterminds/semver" "github.com/antonmedv/expr" "github.com/blackfireio/osinfo" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" + // goccyyaml "github.com/goccy/go-yaml" + + // "github.com/k0kubun/pp" "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" @@ -49,6 +52,7 @@ func validateDataSource(opaqueDS DataSourceItem) error { return nil } + // formally validate YAML commonDS := configuration.DataSourceCommonCfg{} @@ -68,6 +72,7 @@ func validateDataSource(opaqueDS DataSourceItem) error { return fmt.Errorf("source is empty") } + // source must be known ds := acquisition.GetDataSourceIface(commonDS.Source) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index adb5f1d436c..4f6ef0c3375 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -10,7 +10,6 @@ import ( "github.com/lithammer/dedent" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/pkg/csstring" "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -118,16 +117,16 @@ func TestVersionCheck(t *testing.T) { {"1", ">1", false, ""}, {"1", ">=1", true, ""}, {"1.0", "<1.0", false, ""}, - {"1", "<1", false, ""}, - {"1.3.5", "1.3", true, ""}, + {"1", "<1", true, ""}, // XXX why? + {"1.3.5", "1.3", false, ""}, // XXX ok? {"1.0", "<1.0", false, ""}, {"1.0", "<=1.0", true, ""}, {"2", ">1, <3", true, ""}, {"2", "<=2, >=2.2", false, ""}, {"2.3", "~2", true, ""}, {"2.3", "=2", true, ""}, - {"1.1.1", "=1.1", true, ""}, - {"1.1.1", "1.1", true, ""}, + {"1.1.1", "=1.1", false, ""}, + {"1.1.1", "1.1", false, ""}, {"1.1", "!=1.1.1", true, ""}, {"1.1", "~1.1.1", false, ""}, {"1.1.1", "~1.1", true, ""}, @@ -137,7 +136,7 @@ func TestVersionCheck(t *testing.T) { {"19.04", "=19.4", true, ""}, {"19.04", "~19.4", true, ""}, {"1.2.3", "~1.2", true, ""}, - {"1.2.3", "!=1.2", false, ""}, + {"1.2.3", "!=1.2", true, ""}, {"1.2.3", "1.1.1 - 1.3.4", true, ""}, {"1.3.5", "1.1.1 - 1.3.4", false, ""}, {"1.3.5", "=1", true, ""}, @@ -1008,7 +1007,7 @@ func TestDetectDatasourceValidation(t *testing.T) { "DetectYaml": detectYaml, } - expectedErr, err := csstring.Interpolate(tc.expectedErr, data) + expectedErr, err := cstest.Interpolate(tc.expectedErr, data) require.NoError(err) detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) diff --git a/pkg/cwhub/dataset.go b/pkg/types/dataset.go similarity index 70% rename from pkg/cwhub/dataset.go rename to pkg/types/dataset.go index 848686be69d..2684342a94d 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/types/dataset.go @@ -1,4 +1,4 @@ -package cwhub +package types import ( "fmt" @@ -6,14 +6,24 @@ import ( "net/http" "os" "path" + "time" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/types" ) +type DataSource struct { + SourceURL string `yaml:"source_url"` + DestPath string `yaml:"dest_file"` + Type string `yaml:"type"` + //Control cache strategy on expensive regexps + Cache *bool `yaml:"cache"` + Strategy *string `yaml:"strategy"` + Size *int `yaml:"size"` + TTL *time.Duration `yaml:"ttl"` +} + type DataSet struct { - Data []*types.DataSource `yaml:"data,omitempty"` + Data []*DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -56,7 +66,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*types.DataSource, dataDir string) error { +func GetData(data []*DataSource, dataDir string) error { for _, dataS := range data { destPath := path.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/cwhub/dataset_test.go b/pkg/types/dataset_test.go similarity index 94% rename from pkg/cwhub/dataset_test.go rename to pkg/types/dataset_test.go index 106268c01b6..956e3316f5e 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/types/dataset_test.go @@ -1,4 +1,4 @@ -package cwhub +package types import ( "os" @@ -9,7 +9,7 @@ import ( "github.com/jarcoal/httpmock" ) -func TestDownloadFile(t *testing.T) { +func TestDownladFile(t *testing.T) { examplePath := "./example.txt" defer os.Remove(examplePath) diff --git a/pkg/types/datasource.go b/pkg/types/datasource.go deleted file mode 100644 index 39b83fbaf3f..00000000000 --- a/pkg/types/datasource.go +++ /dev/null @@ -1,16 +0,0 @@ -package types - -import ( - "time" -) - -type DataSource struct { - SourceURL string `yaml:"source_url"` - DestPath string `yaml:"dest_file"` - Type string `yaml:"type"` - //Control cache strategy on expensive regexps - Cache *bool `yaml:"cache"` - Strategy *string `yaml:"strategy"` - Size *int `yaml:"size"` - TTL *time.Duration `yaml:"ttl"` -} diff --git a/pkg/parser/grok_pattern.go b/pkg/types/grok_pattern.go similarity index 99% rename from pkg/parser/grok_pattern.go rename to pkg/types/grok_pattern.go index 5b3204a4201..bb51d2d9be0 100644 --- a/pkg/parser/grok_pattern.go +++ b/pkg/types/grok_pattern.go @@ -1,10 +1,9 @@ -package parser +package types import ( "time" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/grokky" ) diff --git a/pkg/types/ip.go b/pkg/types/ip.go index 5e4d7734f2d..647fb4a6303 100644 --- a/pkg/types/ip.go +++ b/pkg/types/ip.go @@ -6,14 +6,13 @@ import ( "math" "net" "strings" + + "github.com/pkg/errors" ) -// LastAddress returns the last address of a network func LastAddress(n net.IPNet) net.IP { - // get the last address by ORing the hostmask and the IP ip := n.IP.To4() if ip == nil { - // IPv6 ip = n.IP return net.IP{ ip[0] | ^n.Mask[0], ip[1] | ^n.Mask[1], ip[2] | ^n.Mask[2], @@ -36,7 +35,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { if strings.Contains(anyIP, "/") { _, net, err := net.ParseCIDR(anyIP) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("while parsing range %s: %w", anyIP, err) + return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing range %s", anyIP) } return Range2Ints(*net) } @@ -48,7 +47,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { sz, start, end, err := IP2Ints(ip) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("while parsing ip %s: %w", anyIP, err) + return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing ip %s", anyIP) } return sz, start, end, start, end, nil @@ -59,12 +58,12 @@ func Range2Ints(network net.IPNet) (int, int64, int64, int64, int64, error) { szStart, nwStart, sfxStart, err := IP2Ints(network.IP) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("converting first ip in range: %w", err) + return -1, 0, 0, 0, 0, errors.Wrap(err, "converting first ip in range") } lastAddr := LastAddress(network) szEnd, nwEnd, sfxEnd, err := IP2Ints(lastAddr) if err != nil { - return -1, 0, 0, 0, 0, fmt.Errorf("transforming last address of range: %w", err) + return -1, 0, 0, 0, 0, errors.Wrap(err, "transforming last address of range") } if szEnd != szStart { return -1, 0, 0, 0, 0, fmt.Errorf("inconsistent size for range first(%d) and last(%d) ip", szStart, szEnd) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 0485db59eaf..caed6961eee 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,9 +2,13 @@ package types import ( "bufio" + "bytes" + "encoding/gob" "fmt" + "io" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -36,9 +40,19 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level if compress != nil { _compress = *compress } + /*cf. https://github.com/natefinch/lumberjack/issues/82 + let's create the file beforehand w/ the right perms */ + fname := cfgFolder + "/crowdsec.log" + // check if file exists + _, err := os.Stat(fname) + // create file if not exists, purposefully ignore errors + if os.IsNotExist(err) { + file, _ := os.OpenFile(fname, os.O_RDWR|os.O_CREATE, 0600) + file.Close() + } LogOutput = &lumberjack.Logger{ - Filename: filepath.Join(cfgFolder, "crowdsec.log"), + Filename: fname, MaxSize: _maxsize, MaxBackups: _maxfiles, MaxAge: _maxage, @@ -68,6 +82,19 @@ func ConfigureLogger(clog *log.Logger) error { return nil } +func Clone(a, b interface{}) error { + buff := new(bytes.Buffer) + enc := gob.NewEncoder(buff) + dec := gob.NewDecoder(buff) + if err := enc.Encode(a); err != nil { + return fmt.Errorf("failed cloning %T", a) + } + if err := dec.Decode(b); err != nil { + return fmt.Errorf("failed cloning %T", b) + } + return nil +} + func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") { @@ -88,6 +115,67 @@ func ParseDuration(d string) (time.Duration, error) { return duration, nil } +/*help to copy the file, ioutil doesn't offer the feature*/ + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +/*copy the file, ioutile doesn't offer the feature*/ +func CopyFile(sourceSymLink, destinationFile string) (err error) { + sourceFile, err := filepath.EvalSymlinks(sourceSymLink) + if err != nil { + log.Infof("Not a symlink : %s", err) + sourceFile = sourceSymLink + } + + sourceFileStat, err := os.Stat(sourceFile) + if err != nil { + return + } + if !sourceFileStat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) + } + destinationFileStat, err := os.Stat(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(destinationFileStat.Mode().IsRegular()) { + return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) + } + if os.SameFile(sourceFileStat, destinationFileStat) { + return + } + } + if err = os.Link(sourceFile, destinationFile); err != nil { + err = copyFileContents(sourceFile, destinationFile) + } + return +} + func UtcNow() time.Time { return time.Now().UTC() } @@ -105,3 +193,11 @@ func GetLineCountForFile(filepath string) int { } return lc } + +// from https://github.com/acarl005/stripansi +var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") + +func StripAnsiString(str string) string { + // the byte version doesn't strip correctly + return reStripAnsi.ReplaceAllString(str, "") +} diff --git a/plugins/notifications/dummy/Makefile b/plugins/notifications/dummy/Makefile index d45d6f19844..612ec6c864c 100644 --- a/plugins/notifications/dummy/Makefile +++ b/plugins/notifications/dummy/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/Makefile b/plugins/notifications/email/Makefile index ae548af0a38..a386625acae 100644 --- a/plugins/notifications/email/Makefile +++ b/plugins/notifications/email/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index bfeb5b02331..549249467ed 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -13,16 +13,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index 8077850dea6..ace03e73be5 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/xhit/go-simple-mail/v2 v2.10.0 h1:nib6RaJ4qVh5HD9UE9QJqnUZyWp3upv+Z6CFxaMj0V8= github.com/xhit/go-simple-mail/v2 v2.10.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -116,15 +117,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/http/Makefile b/plugins/notifications/http/Makefile index 56f49077262..44ee8c58fd5 100644 --- a/plugins/notifications/http/Makefile +++ b/plugins/notifications/http/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index 5f599674efb..a09f74276ce 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index c7ba837554f..e160060b10e 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -114,15 +115,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/slack/Makefile b/plugins/notifications/slack/Makefile index f43303eb882..e950eba92d3 100644 --- a/plugins/notifications/slack/Makefile +++ b/plugins/notifications/slack/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index f8ef3c801d7..92640aad198 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -13,18 +13,18 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index d28c26c3f4c..d5f30d5e700 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -66,13 +66,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -92,7 +93,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -123,15 +124,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/splunk/Makefile b/plugins/notifications/splunk/Makefile index a7f04f4d0fe..a49c87bd647 100644 --- a/plugins/notifications/splunk/Makefile +++ b/plugins/notifications/splunk/Makefile @@ -16,3 +16,8 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index e2d45c36577..04e4c3ccca1 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index c7ba837554f..e160060b10e 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,13 +62,14 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -83,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -114,15 +115,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index a57492eea81..246c7b181de 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -12,6 +12,8 @@ Patch0: crowdsec.unit.patch Patch1: user.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRequires: git +BuildRequires: make BuildRequires: systemd Requires: crontabs %{?fc33:BuildRequires: systemd-rpm-macros} @@ -25,6 +27,7 @@ Requires: crontabs %define version_number %(echo $VERSION) %define releasever %(echo $RELEASEVER) +%global local_version v%{version_number}-%{releasever}-rpm %global name crowdsec %global __mangle_shebangs_exclude_from /usr/bin/env @@ -35,6 +38,7 @@ Requires: crontabs %patch1 %build +BUILD_VERSION=%{local_version} make build sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config/config.yaml %install diff --git a/test/README.md b/test/README.md index 7f34bd3dbef..bbc40a38831 100644 --- a/test/README.md +++ b/test/README.md @@ -56,6 +56,9 @@ architectures. ## pre-requisites - `git submodule init; git submodule update` + - `go install github.com/cloudflare/cfssl/cmd/cfssl@latest` + - `go install github.com/cloudflare/cfssl/cmd/cfssljson@latest` + - `go install github.com/mikefarah/yq/v4@latest` - `base64` - `bash>=4.4` - `curl` @@ -86,7 +89,7 @@ In BATS, you write tests in the form of Bash functions that have unique descriptions (the name of the test). You can do most things that you can normally do in a shell function. If there is any error condition, the test fails. A set of functions is provided to implement assertions, and a mechanism -of `setup`/`teardown` is provided at the level of individual tests (functions) +of `setup`/`teardown` is provided a the level of individual tests (functions) or group of tests (files). The stdout/stderr of the commands within the test function are captured by @@ -126,6 +129,11 @@ included in a larger test suite. The TAP specification is pretty minimalist and some glue may be needed. +Other tools that you can find useful: + + - [mikefarah/yq](https://github.com/mikefarah/yq) - to parse and update YAML files on the fly + - [aliou/bats.vim](https://github.com/aliou/bats.vim) - for syntax highlighting (use bash otherwise) + # setup and teardown If you have read the bats-core tutorial linked above, you are aware of the diff --git a/test/ansible/debug_tools.yml b/test/ansible/debug_tools.yml index 769a973fe95..15baa7cab33 100644 --- a/test/ansible/debug_tools.yml +++ b/test/ansible/debug_tools.yml @@ -14,5 +14,3 @@ - zsh-autosuggestions - zsh-syntax-highlighting - zsh-theme-powerlevel9k - when: - - ansible_facts.os_family == "Debian" diff --git a/test/ansible/provision_dependencies.yml b/test/ansible/provision_dependencies.yml index bcfe8fccafb..891bcc16e11 100644 --- a/test/ansible/provision_dependencies.yml +++ b/test/ansible/provision_dependencies.yml @@ -13,8 +13,6 @@ - crowdsecurity.testing.git - crowdsecurity.testing.gcc - crowdsecurity.testing.gnu_make - - crowdsecurity.testing.pkg_config - - crowdsecurity.testing.re2 - crowdsecurity.testing.bats_requirements - name: "Install Postgres" diff --git a/test/ansible/requirements.yml b/test/ansible/requirements.yml index 70f8ca394db..b1a28b70a02 100644 --- a/test/ansible/requirements.yml +++ b/test/ansible/requirements.yml @@ -11,11 +11,11 @@ roles: - src: gantsign.golang collections: - - name: ansible.posix - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git type: git - version: v0.0.4 + version: main # - name: crowdsecurity.testing # source: ../../../crowdsecurity.testing # type: dir + diff --git a/test/ansible/roles/make_fixture/tasks/main.yml b/test/ansible/roles/make_fixture/tasks/main.yml index 305cec3a697..39f3e1785b9 100644 --- a/test/ansible/roles/make_fixture/tasks/main.yml +++ b/test/ansible/roles/make_fixture/tasks/main.yml @@ -1,6 +1,5 @@ # vim: set ft=yaml.ansible: --- - - name: "Set make_cmd = make (!bsd)" ansible.builtin.set_fact: make_cmd: make @@ -18,8 +17,7 @@ block: - name: "Make bats-build bats-fixture" ansible.builtin.command: - # static build and we don't have to mess with LD_LIBRARY_PATH - cmd: "{{ make_cmd }} bats-build bats-fixture BUILD_STATIC=1" + cmd: "{{ make_cmd }} bats-build bats-fixture" chdir: "{{ ansible_env.HOME }}/crowdsec" creates: "{{ ansible_env.HOME }}/crowdsec/test/local-init/init-config-data.tar" environment: diff --git a/test/ansible/vagrant/alma-8/Vagrantfile b/test/ansible/vagrant/alma-8/Vagrantfile index e246c77ff48..4b42adb3a06 100644 --- a/test/ansible/vagrant/alma-8/Vagrantfile +++ b/test/ansible/vagrant/alma-8/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - sudo dnf config-manager --set-enabled powertools - SHELL end common = '../common' diff --git a/test/ansible/vagrant/alma-9/Vagrantfile b/test/ansible/vagrant/alma-9/Vagrantfile index 9c3d1b67ccf..0ac3e5fe3d5 100644 --- a/test/ansible/vagrant/alma-9/Vagrantfile +++ b/test/ansible/vagrant/alma-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install kitty-terminfo - sudo dnf config-manager --set-enabled crb - SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-7/Vagrantfile b/test/ansible/vagrant/centos-7/Vagrantfile index fd207359f5b..d7ac021d21d 100644 --- a/test/ansible/vagrant/centos-7/Vagrantfile +++ b/test/ansible/vagrant/centos-7/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'centos/7' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-8/Vagrantfile b/test/ansible/vagrant/centos-8/Vagrantfile index a88d621c442..24c37ada97d 100644 --- a/test/ansible/vagrant/centos-8/Vagrantfile +++ b/test/ansible/vagrant/centos-8/Vagrantfile @@ -1,11 +1,7 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'generic/centos8s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - sudo dnf config-manager --set-enabled powertools - SHELL + config.vm.box = 'centos/stream8' end common = '../common' diff --git a/test/ansible/vagrant/centos-9/Vagrantfile b/test/ansible/vagrant/centos-9/Vagrantfile index 1087225bec2..412354f3d5d 100644 --- a/test/ansible/vagrant/centos-9/Vagrantfile +++ b/test/ansible/vagrant/centos-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/centos9s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core - sudo dnf config-manager --set-enabled crb - SHELL end common = '../common' diff --git a/test/ansible/vagrant/common b/test/ansible/vagrant/common index 4bc237a7e5b..adafa08c560 100644 --- a/test/ansible/vagrant/common +++ b/test/ansible/vagrant/common @@ -1,75 +1,46 @@ # vim: set ft=ruby: # frozen_string_literal: true -def find_ansible_cfg - path = Pathname.new(Dir.pwd) - until path.root? - ansible_cfg = path + 'ansible.cfg' - return path if ansible_cfg.exist? - path = path.parent - end - nil # return nil if not found -end - Vagrant.configure('2') do |config| config.vm.define 'crowdsec' - if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } - unless ENV['DB_BACKEND'] - $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" - exit 1 - end - end - - config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') - config.vm.synced_folder '.', '/vagrant', disabled: true - config.vm.provider :libvirt do |libvirt| libvirt.cpus = 1 libvirt.memory = 1536 end - path = find_ansible_cfg - if !path - puts "ansible.cfg not found" - end + config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.provision 'ansible' do |ansible| # ansible.verbose = 'vvvv' - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'run_all.yml').to_s - ansible.compatibility_mode = "2.0" + ansible.config_file = '../../ansible.cfg' + ansible.playbook = '../../run_all.yml' end # same as above, to run the steps separately - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'provision_dependencies.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |provdep| + # provdep.config_file = '../../ansible.cfg' + # provdep.playbook = '../../provision_dependencies.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'provision_test_suite.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |provtest| + # provtest.config_file = '../../ansible.cfg' + # provtest.playbook = '../../provision_test_suite.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'install_binary_package.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |preptest| + # preptest.config_file = '../../ansible.cfg' + # preptest.playbook = '../../install_binary_package.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'prepare_tests.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |preptest| + # preptest.config_file = '../../ansible.cfg' + # preptest.playbook = '../../prepare_tests.yml' # end - # config.vm.provision 'ansible' do |ansible| - # ansible.config_file = (path + 'ansible.cfg').to_s - # ansible.playbook = (path + 'run_tests.yml').to_s - # ansible.compatibility_mode = "2.0" + # config.vm.provision 'ansible' do |runtests| + # runtests.config_file = '../../ansible.cfg' + # runtests.playbook = '../../run_tests.yml' # end end diff --git a/test/ansible/vagrant/debian-10-buster/Vagrantfile b/test/ansible/vagrant/debian-10-buster/Vagrantfile index 8d0a04eaf61..2b1a4e2dac6 100644 --- a/test/ansible/vagrant/debian-10-buster/Vagrantfile +++ b/test/ansible/vagrant/debian-10-buster/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/buster64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile index fbd2ca41eb4..9166427cb3f 100644 --- a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile +++ b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/bullseye64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile deleted file mode 100644 index 68a33a94783..00000000000 --- a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bookworm64' - config.vm.provision "shell", inline: <<-SHELL - # sudo apt install -y kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/debian-9-stretch/Vagrantfile b/test/ansible/vagrant/debian-9-stretch/Vagrantfile index d40b1640cb8..4c4e39cf734 100644 --- a/test/ansible/vagrant/debian-9-stretch/Vagrantfile +++ b/test/ansible/vagrant/debian-9-stretch/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/stretch64' - config.vm.provision "shell", inline: <<-SHELL - sudo sed -i s/httpredir.debian.org/archive.debian.org/g /etc/apt/sources.list - SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-testing/Vagrantfile b/test/ansible/vagrant/debian-testing/Vagrantfile index 6cd2667a960..5e3b68e5411 100644 --- a/test/ansible/vagrant/debian-testing/Vagrantfile +++ b/test/ansible/vagrant/debian-testing/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/testing64' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile index 975d39ea06a..0d34ea126ce 100644 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile @@ -2,10 +2,19 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/hardenedbsd13' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg install python3 - SHELL -end + config.vm.define 'crowdsec' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 1 + libvirt.memory = 1536 + end -common = '../../common' -load common if File.exist?(common) + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = '../../../ansible.cfg' + ansible.playbook = '../../../run_all.yml' + end +end diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap new file mode 100755 index 00000000000..370b1b68e38 --- /dev/null +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo pkg install python3 diff --git a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile deleted file mode 100644 index 1a1fc342164..00000000000 --- a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/openbsd6' - # config.vm.box_version = '4.2.16' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg_add python py3-pip gcc openssl-1.0.2up3 gtar-1.34 - # sudo pkg_add -u - # sudo pkg_add kitty - SHELL -end - -common = '../../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-6/skip b/test/ansible/vagrant/experimental/openbsd-6/skip deleted file mode 100755 index 18d4c60f407..00000000000 --- a/test/ansible/vagrant/experimental/openbsd-6/skip +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -die() { - echo "$@" >&2 - exit 1 -} - -[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution" -exit 0 diff --git a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile index 1c11a94dcac..d7f9801e12b 100644 --- a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile +++ b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile @@ -2,13 +2,19 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/openbsd7' - # config.vm.box_version = '4.2.16' - config.vm.provision "shell", inline: <<-SHELL - sudo pkg_add python-3.9.16 py3-pip gcc-11.2.0p3 openssl-3.0.8 gtar-1.34 - # sudo pkg_add -u - # sudo pkg_add kitty - SHELL -end + config.vm.define 'crowdsec' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 1 + libvirt.memory = 1536 + end -common = '../../common' -load common if File.exist?(common) + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = '../../../ansible.cfg' + ansible.playbook = '../../../run_all.yml' + end +end diff --git a/test/ansible/vagrant/experimental/openbsd-7/bootstrap b/test/ansible/vagrant/experimental/openbsd-7/bootstrap new file mode 100755 index 00000000000..3b2480d3b8b --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-7/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh +unset IFS +set -euf + +sudo pkg_add -u +sudo pkg_add python-3.9.13 py3-pip gcc-11.2.0p2 openssl-3.0.3p0 gtar-1.34 truncate-5.2.1 diff --git a/test/ansible/vagrant/fedora-33/Vagrantfile b/test/ansible/vagrant/fedora-33/Vagrantfile index df6f06944ae..49b5ee990a0 100644 --- a/test/ansible/vagrant/fedora-33/Vagrantfile +++ b/test/ansible/vagrant/fedora-33/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/33-cloud-base" config.vm.box = 'generic/fedora33' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-34/Vagrantfile b/test/ansible/vagrant/fedora-34/Vagrantfile index db2db8d0879..1d172c9c7fe 100644 --- a/test/ansible/vagrant/fedora-34/Vagrantfile +++ b/test/ansible/vagrant/fedora-34/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/34-cloud-base" config.vm.box = 'generic/fedora34' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-35/Vagrantfile b/test/ansible/vagrant/fedora-35/Vagrantfile index c526043405c..f1173076497 100644 --- a/test/ansible/vagrant/fedora-35/Vagrantfile +++ b/test/ansible/vagrant/fedora-35/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = 'fedora/35-cloud-base' config.vm.box = 'generic/fedora35' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-36/Vagrantfile b/test/ansible/vagrant/fedora-36/Vagrantfile index 0cd4fafd023..ef80f514c9e 100644 --- a/test/ansible/vagrant/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/fedora-36/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/36-cloud-base" config.vm.box = 'generic/fedora36' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-37/Vagrantfile b/test/ansible/vagrant/fedora-37/Vagrantfile deleted file mode 100644 index 0eb8978e349..00000000000 --- a/test/ansible/vagrant/fedora-37/Vagrantfile +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/fedora37' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/fedora-38/Vagrantfile b/test/ansible/vagrant/fedora-38/Vagrantfile deleted file mode 100644 index 0e5bf79edc7..00000000000 --- a/test/ansible/vagrant/fedora-38/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = "fedora/38-cloud-base" - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/freebsd-12/Vagrantfile b/test/ansible/vagrant/freebsd-12/Vagrantfile index b500a64d99a..33e6b473f7a 100644 --- a/test/ansible/vagrant/freebsd-12/Vagrantfile +++ b/test/ansible/vagrant/freebsd-12/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd12' - config.vm.provision "shell", inline: <<-SHELL - pkg install -y gtar - SHELL end common = '../common' diff --git a/test/ansible/vagrant/freebsd-13/Vagrantfile b/test/ansible/vagrant/freebsd-13/Vagrantfile index f416ad67793..851c04254b1 100644 --- a/test/ansible/vagrant/freebsd-13/Vagrantfile +++ b/test/ansible/vagrant/freebsd-13/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd13' - config.vm.provision "shell", inline: <<-SHELL - pkg install -y gtar - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-7/Vagrantfile b/test/ansible/vagrant/oracle-7/Vagrantfile index bd435ca5436..638a6123be2 100644 --- a/test/ansible/vagrant/oracle-7/Vagrantfile +++ b/test/ansible/vagrant/oracle-7/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle7' - config.vm.provision "shell", inline: <<-SHELL - sudo yum-config-manager --enable ol7_optional_latest - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-8/Vagrantfile b/test/ansible/vagrant/oracle-8/Vagrantfile index 6744ea87d4e..425ad5e1661 100644 --- a/test/ansible/vagrant/oracle-8/Vagrantfile +++ b/test/ansible/vagrant/oracle-8/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled ol8_codeready_builder - SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-9/Vagrantfile b/test/ansible/vagrant/oracle-9/Vagrantfile index 8dcb4a13ba7..d4e3f618fba 100644 --- a/test/ansible/vagrant/oracle-9/Vagrantfile +++ b/test/ansible/vagrant/oracle-9/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled ol9_codeready_builder - SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-8/Vagrantfile b/test/ansible/vagrant/rocky-8/Vagrantfile index 27caf80c700..c7315cc6718 100644 --- a/test/ansible/vagrant/rocky-8/Vagrantfile +++ b/test/ansible/vagrant/rocky-8/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky8' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled powertools - sudo dnf -y install kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-9/Vagrantfile b/test/ansible/vagrant/rocky-9/Vagrantfile index a4d06e4a9a3..0adb3ab51b8 100644 --- a/test/ansible/vagrant/rocky-9/Vagrantfile +++ b/test/ansible/vagrant/rocky-9/Vagrantfile @@ -2,10 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky9' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf config-manager --set-enabled crb - sudo dnf -y install kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile index 3af88099476..86646ee7ab0 100644 --- a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu1604' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile index 5e25a2ee80c..70a77806bd6 100644 --- a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile @@ -3,8 +3,6 @@ Vagrant.configure('2') do |config| # the official boxes only supports virtualbox config.vm.box = 'generic/ubuntu1804' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile index ea5b339078a..0006ae9267d 100644 --- a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile @@ -2,9 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2004' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y kitty-terminfo - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile index 9e17f71fb6d..c0ccee54bf9 100644 --- a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile @@ -2,8 +2,6 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.provision "shell", inline: <<-SHELL - SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile deleted file mode 100644 index 6c15b0a1e30..00000000000 --- a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'generic/ubuntu2210' - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile deleted file mode 100644 index f40fb7bd59d..00000000000 --- a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'bento/ubuntu-23.04' - config.vm.provision "shell", inline: <<-SHELL - SHELL -end - -common = '../common' -load common if File.exist?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/Vagrantfile b/test/ansible/vagrant/wizard/centos-8/Vagrantfile index 9db09a4ce01..2df31a3920a 100644 --- a/test/ansible/vagrant/wizard/centos-8/Vagrantfile +++ b/test/ansible/vagrant/wizard/centos-8/Vagrantfile @@ -1,13 +1,38 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'generic/centos8s' - config.vm.provision "shell", inline: <<-SHELL - sudo dnf -y install dnf-plugins-core kitty-terminfo - dnf config-manager --set-enabled powertools - # sudo dnf -y update - SHELL -end + config.vm.box = 'centos/stream8' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests -common = '../common' -load common if File.exists?(common) + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/centos-8/bootstrap b/test/ansible/vagrant/wizard/centos-8/bootstrap new file mode 100755 index 00000000000..b33ad9c881a --- /dev/null +++ b/test/ansible/vagrant/wizard/centos-8/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/common b/test/ansible/vagrant/wizard/common deleted file mode 100644 index be1820914c2..00000000000 --- a/test/ansible/vagrant/wizard/common +++ /dev/null @@ -1,67 +0,0 @@ -# vim: set ft=ruby: -# frozen_string_literal: true - -def find_ansible_cfg - path = Pathname.new(Dir.pwd) - until path.root? - ansible_cfg = path + 'ansible.cfg' - return path if ansible_cfg.exist? - path = path.parent - end - nil # return nil if not found -end - -Vagrant.configure('2') do |config| - config.vm.define 'wizard' - - if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } - unless ENV['DB_BACKEND'] - $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" - exit 1 - end - end - - config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - path = find_ansible_cfg - if !path - puts "ansible.cfg not found" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'provision_dependencies.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'provision_test_suite.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'prepare_tests.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'debug_tools.yml').to_s - ansible.compatibility_mode = "2.0" - end - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = (path + 'ansible.cfg').to_s - ansible.playbook = (path + 'run_wizard_tests.yml').to_s - ansible.compatibility_mode = "2.0" - end - -end diff --git a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile deleted file mode 100644 index 3b10b312d0d..00000000000 --- a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/buster64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile deleted file mode 100644 index 6dd7bb2fc9c..00000000000 --- a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bullseye64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile deleted file mode 100644 index 5ccf234eb3e..00000000000 --- a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bookworm64' - config.vm.provision "shell", inline: <<-SHELL - sudo apt update - sudo apt install -y aptitude kitty-terminfo - SHELL -end - -common = '../common' -load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile new file mode 100644 index 00000000000..1d6993a4e97 --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bullseye64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile new file mode 100644 index 00000000000..d4380a94526 --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/buster64' + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/debian-buster/bootstrap b/test/ansible/vagrant/wizard/debian-buster/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-buster/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile index 969a8e70c87..09ee8a39dc9 100644 --- a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile @@ -2,10 +2,37 @@ Vagrant.configure('2') do |config| config.vm.box = 'fedora/36-cloud-base' - config.vm.provision "shell", inline: <<-SHELL - # sudo dnf -y update - SHELL -end + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests -common = '../common' -load common if File.exists?(common) + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end + + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/fedora-36/bootstrap b/test/ansible/vagrant/wizard/fedora-36/bootstrap new file mode 100755 index 00000000000..b33ad9c881a --- /dev/null +++ b/test/ansible/vagrant/wizard/fedora-36/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile index c13d2f9468e..933dabab557 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile @@ -2,10 +2,42 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y aptitude kitty-terminfo - SHELL -end + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end -common = '../common' -load common if File.exists?(common) + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap new file mode 100755 index 00000000000..6a5df521a60 --- /dev/null +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile index d0e2e3cdaa8..c3e8759aa6f 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile @@ -2,10 +2,42 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2210' - config.vm.provision "shell", inline: <<-SHELL - sudo apt install -y aptitude kitty-terminfo - SHELL -end + config.vm.define 'wizard' + + config.vm.provision 'shell', path: 'bootstrap' + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + config.vm.synced_folder '.', '/vagrant', disabled: true + + # install the dependencies for functional tests + + config.vm.provision 'ansible' do |provdep| + provdep.config_file = '../../../ansible.cfg' + provdep.playbook = '../../../provision_dependencies.yml' + end -common = '../common' -load common if File.exists?(common) + config.vm.provision 'ansible' do |provtest| + provtest.config_file = '../../../ansible.cfg' + provtest.playbook = '../../../provision_test_suite.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../prepare_tests.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../debug_tools.yml' + end + + config.vm.provision 'ansible' do |preptest| + preptest.config_file = '../../../ansible.cfg' + preptest.playbook = '../../../run_wizard_tests.yml' + end + +end diff --git a/test/ansible/vars/go.yml b/test/ansible/vars/go.yml index 929e412cd2c..683b4fbbe80 100644 --- a/test/ansible/vars/go.yml +++ b/test/ansible/vars/go.yml @@ -1,5 +1,5 @@ # vim: set ft=yaml.ansible: --- -golang_version: "1.20.5" +golang_version: "1.20.4" golang_install_dir: "/opt/go/{{ golang_version }}" diff --git a/test/bats-detect/openresty-deb.bats b/test/bats-detect/openresty-deb.bats index 0c8bc3c9a0b..c1e91949df3 100644 --- a/test/bats-detect/openresty-deb.bats +++ b/test/bats-detect/openresty-deb.bats @@ -37,13 +37,8 @@ setup() { run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/openresty.gpg < <(output) run -0 sudo tee <<< "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openresty.gpg] http://openresty.org/package/ubuntu $(lsb_release -sc) main" /etc/apt/sources.list.d/openresty.list else - release="$(lsb_release -sc)" - # Debian 12 package is not available as of 2023-07-3 - if [[ "$release" == "bookworm" ]]; then - release="bullseye" - fi run -0 sudo apt-key add - < <(output) - run -0 sudo tee <<< "deb http://openresty.org/package/debian $release openresty" /etc/apt/sources.list.d/openresty.list + run -0 sudo tee <<< "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" /etc/apt/sources.list.d/openresty.list fi run -0 deb-update run -0 deb-install openresty diff --git a/test/bats-detect/openresty-rpm.bats b/test/bats-detect/openresty-rpm.bats index d4c3661bc76..6fc0a8a0975 100644 --- a/test/bats-detect/openresty-rpm.bats +++ b/test/bats-detect/openresty-rpm.bats @@ -34,14 +34,10 @@ setup() { run -0 rpm-install redhat-lsb-core if [[ "$(lsb_release -is)" == "Fedora" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/fedora/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" - elif [[ "$(lsb_release -is)" == CentOS* ]]; then # must match CentOSStream + elif [[ "$(lsb_release -is)" == "CentOS" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/centos/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" fi - run sudo dnf check-update - # 0 = up to date, 100 = updates available - if [[ "$status" -ne 0 ]] && [[ "$status" -ne 100 ]]; then - fail "dnf check-update failed with status $status" - fi + run -0 sudo dnf check-update run -0 rpm-install openresty run -0 sudo systemctl enable openresty.service } diff --git a/test/bats.mk b/test/bats.mk index 4eb7abcbfb7..65bb4a28608 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -10,7 +10,7 @@ ifdef PACKAGE_TESTING INIT_BACKEND = systemd CONFIG_BACKEND = global else - # LOCAL_DIR contains a local instance of crowdsec, complete with + # LOCAL_DIR will contain contains a local instance of crowdsec, complete with # configuration and data LOCAL_DIR = $(TEST_DIR)/local BIN_DIR = $(LOCAL_DIR)/bin @@ -51,7 +51,6 @@ export CONFIG_BACKEND="$(CONFIG_BACKEND)" export PACKAGE_TESTING="$(PACKAGE_TESTING)" export TEST_COVERAGE="$(TEST_COVERAGE)" export GOCOVERDIR="$(TEST_DIR)/coverage" -export PATH="$(TEST_DIR)/tools:$(PATH)" endef bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index a1a2861f6df..75b29033e6c 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -151,10 +151,9 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" # check warning - assert_stderr --partial "no acquisition file found" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" + assert_stderr_line --partial "no acquisition file found" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -167,56 +166,19 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" # check warning - assert_stderr --partial "no acquisition_path or acquisition_dir specified" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" + assert_stderr_line --partial "no acquisition_path or acquisition_dir specified" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') mkdir -p "$ACQUIS_DIR" - mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml + touch "$ACQUIS_DIR"/foo.yaml rune -124 timeout 2s "${CROWDSEC}" - - # now, if foo.yaml is empty instead, there won't be valid datasources. - - cat /dev/null >"$ACQUIS_DIR"/foo.yaml - - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" -} - -@test "crowdsec (disabled datasources)" { - config_set '.common.log_media="stdout"' - - # a datasource cannot run - missing journalctl command - - ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') - mkdir -p "$ACQUIS_DIR" - cat >"$ACQUIS_DIR"/foo.yaml <<-EOT - source: journalctl - journalctl_filter: - - "_SYSTEMD_UNIT=ssh.service" - labels: - type: syslog - EOT - - rune -124 timeout 2s env PATH='' "${CROWDSEC}" - #shellcheck disable=SC2016 - assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' - - # if all datasources are disabled, crowdsec should exit - - ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') - rm -f "$ACQUIS_YAML" - config_set '.crowdsec_service.acquisition_path=""' - - rune -1 timeout 2s env PATH='' "${CROWDSEC}" - assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } - diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index ef825d2a8e6..a01d936b787 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -101,9 +101,9 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) rune -0 cscli config show -o human - assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" - assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX" - assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" + assert_line --regexp ".*- URL\s+: http://127.0.0.1:8080/" + assert_line --regexp ".*- Login\s+: githubciXXXXXXXXXXXXXXXXXXXXXXXX" + assert_line --regexp ".*- Credentials File\s+: .*/local_api_credentials.yaml" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output) @@ -182,7 +182,7 @@ teardown() { @test "cscli - empty LAPI credentials file" { LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') - : > "${LOCAL_API_CREDENTIALS}" + truncate -s 0 "${LOCAL_API_CREDENTIALS}" rune -1 cscli lapi status assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'" diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index f4c9f49e0f7..7015f2c5dbd 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -60,5 +60,5 @@ setup() { ONLINE_API_CREDENTIALS_YAML="$(config_get '.api.server.online_client.credentials_path')" rm "${ONLINE_API_CREDENTIALS_YAML}" rune -1 cscli capi status - assert_stderr --partial "local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" + assert_stderr --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" } diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index 9640e333073..c216067f9e1 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -36,35 +36,35 @@ teardown() { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" } @test "duplicate notification config" { @@ -85,14 +85,14 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" } @test "config.yaml: missing .plugin_config section" { @@ -116,9 +116,9 @@ teardown() { assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" } -@test "unable to run plugin broker: while reading plugin config" { +@test "unable to run local API: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + assert_stderr --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory" } diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index bcb410de979..3499f3e0ed8 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -5,9 +5,6 @@ set -u setup_file() { load "../lib/setup_file.sh" - - TESTDATA="${BATS_TEST_DIRNAME}/testdata/90_decisions" - export TESTDATA } teardown_file() { @@ -59,122 +56,8 @@ teardown() { @test "cscli decisions list, incorrect parameters" { rune -1 cscli decisions list --until toto - assert_stderr --partial 'unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\"' + assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"' rune -1 cscli decisions list --until toto -o json rune -0 jq -c '[.level, .msg]' <(stderr | grep "^{") - assert_output '["fatal","unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\""]' -} - -@test "cscli decisions import" { - # required input - rune -1 cscli decisions import - assert_stderr --partial 'required flag(s) \"input\" not set"' - - # unsupported format - rune -1 cscli decisions import -i - <<<'value\n5.6.7.8' --format xml - assert_stderr --partial "invalid format 'xml', expected one of 'json', 'csv', 'values'" - - # invalid defaults - rune -1 cscli decisions import --duration "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--duration cannot be empty" - rune -1 cscli decisions import --scope "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--scope cannot be empty" - rune -1 cscli decisions import --reason "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--reason cannot be empty" - rune -1 cscli decisions import --type "" -i - <<<'value\n5.6.7.8' --format csv - assert_stderr --partial "--type cannot be empty" - - #---------- - # JSON - #---------- - - # import from file - rune -1 cscli decisions import -i "${TESTDATA}/json_decisions" - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - - rune -0 cscli decisions import -i "${TESTDATA}/decisions.json" - assert_stderr --partial "Parsing json" - assert_stderr --partial "Imported 5 decisions" - - # import from stdin - rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") --format json - assert_stderr --partial "Parsing json" - assert_stderr --partial "Imported 5 decisions" - - # invalid json - rune -1 cscli decisions import -i - <<<'{"blah":"blah"}' --format json - assert_stderr --partial 'Parsing json' - assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' - - # json with extra data - rune -1 cscli decisions import -i - <<<'{"values":"1.2.3.4","blah":"blah"}' --format json - assert_stderr --partial 'Parsing json' - assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' - - #---------- - # CSV - #---------- - - # import from file - rune -1 cscli decisions import -i "${TESTDATA}/csv_decisions" - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - - rune -0 cscli decisions import -i "${TESTDATA}/decisions.csv" - assert_stderr --partial 'Parsing csv' - assert_stderr --partial 'Imported 5 decisions' - - # import from stdin - rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") - assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" - rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") --format csv - assert_stderr --partial "Parsing csv" - assert_stderr --partial "Imported 5 decisions" - - # invalid csv - # XXX: improve validation - rune -0 cscli decisions import -i - <<<'value\n1.2.3.4,5.6.7.8' --format csv - assert_stderr --partial 'Parsing csv' - assert_stderr --partial "Imported 0 decisions" - - #---------- - # VALUES - #---------- - - # can use '-' as stdin - rune -0 cscli decisions import -i - --format values <<-EOT - 1.2.3.4 - 1.2.3.5 - 1.2.3.6 - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'Imported 3 decisions' - - rune -0 cscli decisions import -i - --format values <<-EOT - 10.2.3.4 - 10.2.3.5 - 10.2.3.6 - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'Imported 3 decisions' - - rune -1 cscli decisions import -i - --format values <<-EOT - whatever - EOT - assert_stderr --partial 'Parsing values' - assert_stderr --partial 'API error: unable to create alerts: whatever: invalid ip address / range' - - #---------- - # Batch - #---------- - - rune -0 cscli decisions import -i - --format values --batch 2 --debug <<-EOT - 1.2.3.4 - 1.2.3.5 - 1.2.3.6 - EOT - assert_stderr --partial 'Processing chunk of 2 decisions' - assert_stderr --partial 'Processing chunk of 1 decisions' - assert_stderr --partial 'Imported 3 decisions' + assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]' } diff --git a/test/bats/testdata/90_decisions/csv_decisions b/test/bats/testdata/90_decisions/csv_decisions deleted file mode 100644 index 858654b6304..00000000000 --- a/test/bats/testdata/90_decisions/csv_decisions +++ /dev/null @@ -1,6 +0,0 @@ -origin,scope,value,reason,type,duration -cscli,ip,1.6.11.16,manual import from csv,ban,1h -cscli,ip,2.7.12.17,manual import from csv,ban,1h -cscli,ip,3.8.13.18,manual import from csv,ban,1h -cscli,ip,4.9.14.19,manual import from csv,ban,1h -cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.csv b/test/bats/testdata/90_decisions/decisions.csv deleted file mode 100644 index 858654b6304..00000000000 --- a/test/bats/testdata/90_decisions/decisions.csv +++ /dev/null @@ -1,6 +0,0 @@ -origin,scope,value,reason,type,duration -cscli,ip,1.6.11.16,manual import from csv,ban,1h -cscli,ip,2.7.12.17,manual import from csv,ban,1h -cscli,ip,3.8.13.18,manual import from csv,ban,1h -cscli,ip,4.9.14.19,manual import from csv,ban,1h -cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.json b/test/bats/testdata/90_decisions/decisions.json deleted file mode 100644 index 395458c9785..00000000000 --- a/test/bats/testdata/90_decisions/decisions.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "origin": "cscli", - "scope": "ip", - "value": "1.6.11.16", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "2.7.12.17", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "3.8.13.18", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "4.9.14.19", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "5.10.15.20", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - } -] diff --git a/test/bats/testdata/90_decisions/json_decisions b/test/bats/testdata/90_decisions/json_decisions deleted file mode 100644 index 395458c9785..00000000000 --- a/test/bats/testdata/90_decisions/json_decisions +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "origin": "cscli", - "scope": "ip", - "value": "1.6.11.16", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "2.7.12.17", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "3.8.13.18", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "4.9.14.19", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - }, - { - "origin": "cscli", - "scope": "ip", - "value": "5.10.15.20", - "reason": "manual import from csv", - "type": "ban", - "duration": "1h" - } -] diff --git a/test/bin/assert-crowdsec-not-running b/test/bin/assert-crowdsec-not-running index 3171287d0e6..b545ebf0af3 100755 --- a/test/bin/assert-crowdsec-not-running +++ b/test/bin/assert-crowdsec-not-running @@ -1,15 +1,8 @@ #!/usr/bin/env bash is_crowdsec_running() { - case $(uname) in - "Linux") - # ignore processes in containers - PIDS=$(pgrep --ns $$ -x 'crowdsec') - ;; - *) - PIDS=$(pgrep -x 'crowdsec') - ;; - esac + # ignore processes in containers + PIDS=$(pgrep --ns $$ -x 'crowdsec') } # The process can be slow, especially on CI and during test coverage. diff --git a/test/bin/check-requirements b/test/bin/check-requirements index 0563eec01e4..351c0a01b13 100755 --- a/test/bin/check-requirements +++ b/test/bin/check-requirements @@ -54,25 +54,47 @@ check_pkill() { fi } +check_yq() { + # shellcheck disable=SC2016 + howto_install='You can install it with your favorite package manager (including snap) or with "go install github.com/mikefarah/yq/v4@latest" and add ~/go/bin to $PATH.' + if ! command -v yq >/dev/null; then + die "Missing required program 'yq'. ${howto_install}" + fi + if ! (yq --version | grep mikefarah >/dev/null); then + die "yq exists but it's not the one we need (mikefarah/yq). ${howto_install}" + fi +} + check_daemonizer() { if ! command -v daemonize >/dev/null; then die "missing required program 'daemonize' (package 'daemonize' or 'https://github.com/bmc/daemonize')" fi } -echo "Checking requirements..." +check_cfssl() { + # shellcheck disable=SC2016 + howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssl@latest" and add ~/go/bin to $PATH.' + if ! command -v cfssl >/dev/null; then + die "Missing required program 'cfssl'. ${howto_install}" + fi +} -GOBIN=${TEST_DIR}/tools -export GOBIN -go install github.com/mikefarah/yq/v4@latest -go install github.com/cloudflare/cfssl/cmd/cfssl@master -go install github.com/cloudflare/cfssl/cmd/cfssljson@master +check_cfssljson() { + # shellcheck disable=SC2016 + howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssljson@latest" and add ~/go/bin to $PATH.' + if ! command -v cfssljson >/dev/null; then + die "Missing required program 'cfssljson'. ${howto_install}" + fi +} check_bats_core check_curl check_daemonizer +check_cfssl +check_cfssljson check_jq check_nc check_base64 check_python3 +check_yq check_pkill diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index a4231c98edb..22a9bc303bc 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -127,24 +127,6 @@ is_db_sqlite() { } export -f is_db_sqlite -crowdsec_log() { - echo "$(config_get .common.log_dir)"/crowdsec.log -} -export -f crowdsec_log - -truncate_log() { - true > "$(crowdsec_log)" -} -export -f truncate_log - -assert_log() { - local oldout="${output:-}" - output="$(cat "$(crowdsec_log)")" - assert_output "$@" - output="${oldout}" -} -export -f assert_log - # Compare ignoring the key order, and allow "expected" without quoted identifiers. # Preserve the output variable in case the following commands require it. assert_json() { diff --git a/test/tools/.do-not-remove b/test/tools/.do-not-remove deleted file mode 100644 index 3119281709f..00000000000 --- a/test/tools/.do-not-remove +++ /dev/null @@ -1 +0,0 @@ -this directory is populated by test dependencies, and is not checked in git From 8baeb709986de7d59d37d39441374b7d89998852 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 10 Jul 2023 18:00:19 +0200 Subject: [PATCH 027/263] add metrics --- cmd/crowdsec/metrics.go | 9 ++++- pkg/acquisition/modules/waf/utils.go | 2 + pkg/acquisition/modules/waf/waf.go | 57 +++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 8e87eecd037..96db7290fca 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -12,6 +12,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/go-cs-lib/pkg/version" + waf "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -169,7 +170,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, v1.LapiRouteHits, leaky.BucketsCurrentCount, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + waf.WafGlobalParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + ) } else { log.Infof("Loading prometheus collectors") prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, @@ -178,7 +181,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime, leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + waf.WafGlobalParsingHistogram, waf.WafInbandParsingHistogram, waf.WafOutbandParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + ) } } diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 6d2932bf9e3..879557477b1 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -23,6 +24,7 @@ func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) { if rule.Message() == "" { continue } + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) if err != nil { return nil, errors.Wrap(err, "Cannot convert rule match to event") diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index e46e875ebab..51120d8cfed 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/antonmedv/expr" "github.com/corazawaf/coraza/v3" @@ -62,6 +63,49 @@ type WafSource struct { WafRunners []WafRunner } +var WafGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband WAF.", + Name: "cs_waf_inband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_outband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_reqs_total", + Help: "Total events processed by the WAF.", + }, + []string{"source"}, +) + +var WafRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_rule_hits", + Help: "Count of triggered rule, by rule_id and type (inband/outofband).", + }, + []string{"rule_id", "type"}, +) + func (w *WafSource) GetMetrics() []prometheus.Collector { return nil } @@ -362,6 +406,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { log.Infof("Waf Runner is dying") return nil case request := <-r.inChan: + WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() + //measure the time spent in the WAF + startParsing := time.Now() inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID) expTx := inBoundTx.(experimental.FullTransaction) // we use this internal transaction for the expr helpers @@ -457,7 +504,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } } - + //measure the full time spent in the WAF + elapsed := time.Since(startParsing) + WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response if in != nil && response.SendEvents { @@ -473,6 +522,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } + outBandStart := time.Now() // Process outBand outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) expTx = outBandTx.(experimental.FullTransaction) @@ -493,6 +543,11 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { continue } } + //measure the full time spent in the WAF + totalElapsed := time.Since(startParsing) + WafGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(totalElapsed.Seconds()) + elapsed = time.Since(outBandStart) + WafOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) } } } From a6ba0e869c915c7bb4dba94c4095f27e3b0ff15c Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 11 Jul 2023 09:29:17 +0200 Subject: [PATCH 028/263] imp logging --- pkg/acquisition/modules/waf/utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 879557477b1..a6a5b534fe6 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -44,7 +44,11 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, //def needs fixing evt.Stage = "s00-raw" evt.Process = true - log.Infof("SOURCE IP: %+v", rule) + log.WithFields(log.Fields{ + "module": "waf", + "source": rule.ClientIPAddress(), + "id": rule.Rule().ID(), + }).Infof("%s", rule.Message()) //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers CorazaEvent := map[string]interface{}{ From 57547c32c9f3f0d2c5dfda19d26cf33599c708b8 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 13 Jul 2023 16:20:04 +0200 Subject: [PATCH 029/263] Aggregate WAF rules into a single event (#2350) --- pkg/acquisition/modules/waf/utils.go | 122 +++++++++++++------------ pkg/acquisition/modules/waf/waf.go | 36 +++++--- pkg/types/event.go | 129 +++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 75 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index a6a5b534fe6..988e92fa317 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -1,42 +1,18 @@ package wafacquisition import ( - "encoding/json" "fmt" - "strings" "time" - corazatypes "github.com/corazawaf/coraza/v3/types" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/corazawaf/coraza/v3/experimental" + types "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/pkg/errors" + "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) -func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) { - evts := []types.Event{} - if r.Tx == nil { - return nil, fmt.Errorf("tx is nil") - } - for _, rule := range r.Tx.MatchedRules() { - //log.Printf("rule %d", idx) - if rule.Message() == "" { - continue - } - WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() - evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) - if err != nil { - return nil, errors.Wrap(err, "Cannot convert rule match to event") - } - evts = append(evts, evt) - } - - return evts, nil -} - -// Transforms a coraza interruption to a crowdsec event -func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) { +func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -44,39 +20,12 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, //def needs fixing evt.Stage = "s00-raw" evt.Process = true - log.WithFields(log.Fields{ - "module": "waf", - "source": rule.ClientIPAddress(), - "id": rule.Rule().ID(), - }).Infof("%s", rule.Message()) - //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. - //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers - CorazaEvent := map[string]interface{}{ - //core rule info - "rule_type": kind, - "rule_id": rule.Rule().ID(), - //"rule_action": tx.Interruption().Action, - "rule_disruptive": rule.Disruptive(), - "rule_tags": rule.Rule().Tags(), - "rule_file": rule.Rule().File(), - "rule_file_line": rule.Rule().Line(), - "rule_revision": rule.Rule().Revision(), - "rule_secmark": rule.Rule().SecMark(), - "rule_accuracy": rule.Rule().Accuracy(), - - //http contextual infos - "upstream_addr": r.RemoteAddr, - "req_uuid": tx.ID(), - "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], - "uri": rule.URI(), - } - - if tx.Interruption() != nil { - CorazaEvent["rule_action"] = tx.Interruption().Action - } - corazaEventB, err := json.Marshal(CorazaEvent) - if err != nil { - return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) + evt.Parsed = map[string]string{ + "source_ip": r.ClientIP, + "target_host": r.Host, + "target_uri": r.URI, + "method": r.Method, + "req_uuid": r.Tx.ID(), } evt.Line = types.Line{ Time: time.Now(), @@ -85,8 +34,57 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, Process: true, Module: "waf", Src: "waf", - Raw: string(corazaEventB), + Raw: "dummy-waf-data", //we discard empty Line.Raw items :) } + evt.Waap = []map[string]interface{}{} return evt, nil } + +func LogWaapEvent(evt *types.Event) { + log.WithFields(log.Fields{ + "module": "waf", + "source": evt.Parsed["source_ip"], + "target_uri": evt.Parsed["target_uri"], + }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs()) + log.Infof("%s", evt.Waap) +} + +func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { + + if tx.IsInterrupted() { + log.Infof("interrupted() = %t", tx.IsInterrupted()) + log.Infof("interrupted.action = %s", tx.Interruption().Action) + if evt.Meta == nil { + evt.Meta = map[string]string{} + } + evt.Meta["waap_interrupted"] = "1" + evt.Meta["waap_action"] = tx.Interruption().Action + } + log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + for _, rule := range tx.MatchedRules() { + if rule.Message() == "" { + continue + } + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() + + corazaRule := map[string]interface{}{ + "id": rule.Rule().ID(), + "uri": evt.Parsed["uri"], + "rule_type": kind, + "method": evt.Parsed["method"], + "disruptive": rule.Disruptive(), + "tags": rule.Rule().Tags(), + "file": rule.Rule().File(), + "file_line": rule.Rule().Line(), + "revision": rule.Rule().Revision(), + "secmark": rule.Rule().SecMark(), + "accuracy": rule.Rule().Accuracy(), + "msg": rule.Message(), + "severity": rule.Rule().Severity().String(), + } + evt.Waap = append(evt.Waap, corazaRule) + } + + return nil +} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 51120d8cfed..ea314b936db 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -17,6 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -406,6 +407,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { log.Infof("Waf Runner is dying") return nil case request := <-r.inChan: + var evt *types.Event WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() //measure the time spent in the WAF startParsing := time.Now() @@ -458,6 +460,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { in, expTx, err := processReqWithEngine(expTx, request, InBand) request.Tx = expTx + log.Infof("-> %s", spew.Sdump(in)) response := waf.NewResponseRequest(expTx, in, request.UUID, err) @@ -510,16 +513,17 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response if in != nil && response.SendEvents { - // Generate the events for InBand channel - events, err := TxToEvents(request, InBand) + evt = &types.Event{} + *evt, err = EventFromRequest(request) if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue + return fmt.Errorf("cannot create event from waap context : %w", err) } - - for _, evt := range events { - r.outChan <- evt + err = AccumulateTxToEvent(expTx, InBand, evt) + if err != nil { + return fmt.Errorf("cannot convert transaction to event : %w", err) } + LogWaapEvent(evt) + r.outChan <- *evt } outBandStart := time.Now() @@ -533,15 +537,21 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } request.Tx = expTx if expTx != nil && len(expTx.MatchedRules()) > 0 { - events, err := TxToEvents(request, OutOfBand) - log.Infof("Request triggered by WAF, %d events to send", len(events)) - for _, evt := range events { - r.outChan <- evt + //if event was not instantiated after inband processing, do it now + if evt == nil { + *evt, err = EventFromRequest(request) + if err != nil { + return fmt.Errorf("cannot create event from waap context : %w", err) + } } + + err = AccumulateTxToEvent(expTx, InBand, evt) if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue + return fmt.Errorf("cannot convert transaction to event : %w", err) } + LogWaapEvent(evt) + r.outChan <- *evt + } //measure the full time spent in the WAF totalElapsed := time.Since(startParsing) diff --git a/pkg/types/event.go b/pkg/types/event.go index fc8d966abc7..7dc957996e9 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -1,6 +1,7 @@ package types import ( + "regexp" "time" log "github.com/sirupsen/logrus" @@ -14,6 +15,133 @@ const ( OVFLW ) +/* + 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban + 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban + + +any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) + +len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 + +*/ + +type WaapEvent []map[string]interface{} + +func (w WaapEvent) ByID(id int) WaapEvent { + waap := WaapEvent{} + + for _, rule := range w { + if rule["id"] == id { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) GetURI() string { + for _, rule := range w { + return rule["uri"].(string) + } + return "" +} + +func (w WaapEvent) GetMethod() string { + for _, rule := range w { + return rule["method"].(string) + } + return "" +} + +func (w WaapEvent) GetRuleIDs() []int { + ret := make([]int, 0) + for _, rule := range w { + ret = append(ret, rule["id"].(int)) + } + return ret +} + +func (w WaapEvent) ByKind(kind string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + if rule["kind"] == kind { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) Kinds() []string { + ret := make([]string, 0) + for _, rule := range w { + exists := false + for _, val := range ret { + if val == rule["kind"] { + exists = true + break + } + } + if !exists { + ret = append(ret, rule["kind"].(string)) + } + } + return ret +} + +func (w WaapEvent) ByTag(match string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if tag == match { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w WaapEvent) ByTagRx(rx string) WaapEvent { + waap := WaapEvent{} + re := regexp.MustCompile(rx) + if re == nil { + return waap + } + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if re.MatchString(tag) { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent { + log.Infof("%s", w) + wap := WaapEvent{} + for _, rule := range w { + if rule["disruptive"] == is { + wap = append(wap, rule) + } + } + log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap)) + + return wap +} + +func (w WaapEvent) BySeverity(severity string) WaapEvent { + wap := WaapEvent{} + for _, rule := range w { + if rule["severity"] == severity { + wap = append(wap, rule) + } + } + log.Infof("BySeverity(%t) -> %d", severity, len(wap)) + return wap +} + // Event is the structure representing a runtime event (log or overflow) type Event struct { /* is it a log or an overflow */ @@ -39,6 +167,7 @@ type Event struct { StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line + Waap WaapEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"` /* Meta is the only part that will make it to the API - it should be normalized */ Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } From ef4fe8f5d31774b65ea47a93b7f0bfba2eef4cf0 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 13 Jul 2023 16:22:21 +0200 Subject: [PATCH 030/263] merge --- go.mod | 18 ++++++++--------- go.sum | 30 +++++++++++++++------------- pkg/acquisition/modules/waf/utils.go | 4 ++-- pkg/acquisition/modules/waf/waf.go | 9 +++++---- pkg/waf/env.go | 2 +- pkg/waf/request.go | 4 ++-- 6 files changed, 34 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index d64e1c0754b..9fedf14a8d2 100644 --- a/go.mod +++ b/go.mod @@ -48,10 +48,10 @@ require ( github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 - github.com/sirupsen/logrus v1.9.2 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 - golang.org/x/crypto v0.1.0 + golang.org/x/crypto v0.10.0 golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 @@ -70,8 +70,8 @@ require ( github.com/blackfireio/osinfo v1.0.3 github.com/bluele/gcache v0.0.2 github.com/cespare/xxhash/v2 v2.1.2 - github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 github.com/coreos/go-systemd/v22 v22.5.0 + github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5 github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd github.com/goccy/go-yaml v1.9.7 github.com/gofrs/uuid v4.0.0+incompatible @@ -87,7 +87,7 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v0.2.1 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sys v0.7.0 + golang.org/x/sys v0.9.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -145,7 +145,7 @@ require ( github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magefile/mage v1.14.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect @@ -185,10 +185,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -204,5 +204,3 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 - -replace github.com/corazawaf/coraza/v3 => ./coraza diff --git a/go.sum b/go.sum index 8d45067581f..19fcd83317e 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5 h1:zpU5j3EOvH/Hxs3trWwsXp7Vr1T0YP0MIdp89RvVlY8= +github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= @@ -642,8 +644,8 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -849,8 +851,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -1019,8 +1021,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1107,8 +1109,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1210,14 +1212,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1227,8 +1229,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 988e92fa317..8c937592dac 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - "github.com/corazawaf/coraza/v3/experimental" - types "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/coraza/v3/experimental" + "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index ea314b936db..a518b89b41c 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -10,9 +10,9 @@ import ( "time" "github.com/antonmedv/expr" - "github.com/corazawaf/coraza/v3" - "github.com/corazawaf/coraza/v3/experimental" - corazatypes "github.com/corazawaf/coraza/v3/types" + "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/experimental" + corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" @@ -149,7 +149,7 @@ func (w *WafSource) UnmarshalConfig(yamlConfig []byte) error { } func logError(error corazatypes.MatchedRule) { - msg := error.ErrorLog(0) + msg := error.ErrorLog() log.Infof("[logError][%s] %s", error.Rule().Severity(), msg) } @@ -356,6 +356,7 @@ func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, //spew.Dump(tx.MatchedRules()) for _, rule := range tx.MatchedRules() { + log.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) if rule.Message() == "" { continue } diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 797fe5c37a6..3a5de2be501 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -1,6 +1,6 @@ package waf -import "github.com/corazawaf/coraza/v3/experimental" +import "github.com/crowdsecurity/coraza/v3/experimental" type Transaction struct { Tx experimental.FullTransaction diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 29d9ab35f33..a5a0c5ffdf0 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -6,8 +6,8 @@ import ( "net/http" "net/url" - "github.com/corazawaf/coraza/v3/experimental" - corazatypes "github.com/corazawaf/coraza/v3/types" + "github.com/crowdsecurity/coraza/v3/experimental" + corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/google/uuid" ) From f7eaefa5186f89e42c2d38f29b60311a03dee5de Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 18 Jul 2023 18:12:17 +0200 Subject: [PATCH 031/263] up --- pkg/acquisition/modules/waf/utils.go | 11 +++- pkg/types/event.go | 91 ++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 8c937592dac..c0469845ad1 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -26,11 +26,17 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { "target_uri": r.URI, "method": r.Method, "req_uuid": r.Tx.ID(), + "source": "coraza", + + //TBD: + //http_status + //user_agent + } evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "waf"}, + Labels: map[string]string{"type": "coraza-waf"}, Process: true, Module: "waf", Src: "waf", @@ -58,6 +64,9 @@ func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *type if evt.Meta == nil { evt.Meta = map[string]string{} } + evt.Parsed["interrupted"] = "true" + evt.Parsed["action"] = tx.Interruption().Action + evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } diff --git a/pkg/types/event.go b/pkg/types/event.go index 7dc957996e9..a0741c66018 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -1,6 +1,7 @@ package types import ( + "fmt" "regexp" "time" @@ -28,15 +29,33 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 type WaapEvent []map[string]interface{} -func (w WaapEvent) ByID(id int) WaapEvent { - waap := WaapEvent{} +type Field string + +func (f Field) String() string { + return fmt.Sprintf("%s", f) +} +const ( + ID Field = "id" + RuleType Field = "rule_type" + Tags Field = "tags" + File Field = "file" + Confidence Field = "confidence" + Revision Field = "revision" + SecMark Field = "secmark" + Accuracy Field = "accuracy" + Msg Field = "msg" + Severity Field = "severity" + Kind Field = "kind" +) + +// getters +func (w WaapEvent) GetField(field Field) []interface{} { + ret := make([]interface{}, 0) for _, rule := range w { - if rule["id"] == id { - waap = append(waap, rule) - } + ret = append(ret, rule[field.String()]) } - return waap + return ret } func (w WaapEvent) GetURI() string { @@ -61,16 +80,6 @@ func (w WaapEvent) GetRuleIDs() []int { return ret } -func (w WaapEvent) ByKind(kind string) WaapEvent { - waap := WaapEvent{} - for _, rule := range w { - if rule["kind"] == kind { - waap = append(waap, rule) - } - } - return waap -} - func (w WaapEvent) Kinds() []string { ret := make([]string, 0) for _, rule := range w { @@ -88,6 +97,43 @@ func (w WaapEvent) Kinds() []string { return ret } +// filters +func (w WaapEvent) ByID(id int) WaapEvent { + waap := WaapEvent{} + + for _, rule := range w { + if rule["id"] == id { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) ByKind(kind string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + if rule["kind"] == kind { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) ByTags(match []string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + for _, match_tag := range match { + if tag == match_tag { + waap = append(waap, rule) + break + } + } + } + } + return waap +} + func (w WaapEvent) ByTag(match string) WaapEvent { waap := WaapEvent{} for _, rule := range w { @@ -138,7 +184,18 @@ func (w WaapEvent) BySeverity(severity string) WaapEvent { wap = append(wap, rule) } } - log.Infof("BySeverity(%t) -> %d", severity, len(wap)) + log.Infof("BySeverity(%s) -> %d", severity, len(wap)) + return wap +} + +func (w WaapEvent) ByAccuracy(accuracy string) WaapEvent { + wap := WaapEvent{} + for _, rule := range w { + if rule["accuracy"] == accuracy { + wap = append(wap, rule) + } + } + log.Infof("ByAccuracy(%s) -> %d", accuracy, len(wap)) return wap } From e543523ba35535b22b2960d52e060a054b9b64b4 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 10:34:13 +0200 Subject: [PATCH 032/263] update ban remediation --- pkg/acquisition/modules/waf/waf.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index a518b89b41c..435d33e2a33 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -587,7 +587,11 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { if message.Interruption != nil { rw.WriteHeader(http.StatusForbidden) - body, err := json.Marshal(BodyResponse{Action: message.Interruption.Action}) + action := message.Interruption.Action + if action == "deny" { // bouncers understand "ban" and not "deny" + action = "ban" + } + body, err := json.Marshal(BodyResponse{Action: action}) if err != nil { log.Errorf("unable to build response: %s", err) } else { From faf2042258d849c39c3b49ab120a3a47cccf7f8f Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 10:39:16 +0200 Subject: [PATCH 033/263] upate go.mods --- plugins/notifications/http/go.mod | 6 +++--- plugins/notifications/http/go.sum | 14 +++++++------- plugins/notifications/slack/go.mod | 6 +++--- plugins/notifications/slack/go.sum | 14 +++++++------- plugins/notifications/splunk/go.mod | 6 +++--- plugins/notifications/splunk/go.sum | 14 +++++++------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index a09f74276ce..614d5654769 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index e160060b10e..79381ac19cc 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -84,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -125,14 +125,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index 92640aad198..8a99949d595 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -22,9 +22,9 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index d5f30d5e700..f47d069925c 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -93,7 +93,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -111,8 +111,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -134,14 +134,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index 04e4c3ccca1..99de7c6b5da 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -19,9 +19,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index e160060b10e..79381ac19cc 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -84,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -102,8 +102,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -125,14 +125,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From d0af521b9eb804b8a43e981f6426ff0ed8938b31 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 10:45:42 +0200 Subject: [PATCH 034/263] update --- plugins/notifications/email/go.mod | 6 +++--- plugins/notifications/email/go.sum | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index 549249467ed..ba260c5571d 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -20,9 +20,9 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index ace03e73be5..53a3f1ac87e 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -84,7 +84,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/xhit/go-simple-mail/v2 v2.10.0 h1:nib6RaJ4qVh5HD9UE9QJqnUZyWp3upv+Z6CFxaMj0V8= github.com/xhit/go-simple-mail/v2 v2.10.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -104,8 +104,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -127,14 +127,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From d946286e5ca704de22e130678134e9536c959c28 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 11:50:42 +0200 Subject: [PATCH 035/263] remove spew --- pkg/acquisition/modules/waf/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index c0469845ad1..6c1f688cc0b 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -7,7 +7,6 @@ import ( "github.com/crowdsecurity/coraza/v3/experimental" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -70,7 +69,7 @@ func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *type evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } - log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) for _, rule := range tx.MatchedRules() { if rule.Message() == "" { continue From cd5cb55a7e19bbb2e65cbabdeb48011c5d9bc7f0 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 11:57:14 +0200 Subject: [PATCH 036/263] debug --- pkg/acquisition/modules/waf/utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 6c1f688cc0b..f3a56b86a67 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -7,6 +7,7 @@ import ( "github.com/crowdsecurity/coraza/v3/experimental" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" + "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -56,7 +57,7 @@ func LogWaapEvent(evt *types.Event) { } func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { - + log.Infof("TX %s", spew.Sdump(tx)) if tx.IsInterrupted() { log.Infof("interrupted() = %t", tx.IsInterrupted()) log.Infof("interrupted.action = %s", tx.Interruption().Action) From 8ba692b1150c2919804fcaca67e9f785db060879 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 12:02:38 +0200 Subject: [PATCH 037/263] debug --- pkg/acquisition/modules/waf/utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index f3a56b86a67..55e673ba904 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -7,7 +7,6 @@ import ( "github.com/crowdsecurity/coraza/v3/experimental" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -57,7 +56,7 @@ func LogWaapEvent(evt *types.Event) { } func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { - log.Infof("TX %s", spew.Sdump(tx)) + log.Infof("TX %v", &tx) if tx.IsInterrupted() { log.Infof("interrupted() = %t", tx.IsInterrupted()) log.Infof("interrupted.action = %s", tx.Interruption().Action) From 7d8c931d0019ceaf95be5e1d88df76298338c795 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 14:35:02 +0200 Subject: [PATCH 038/263] add loggers --- pkg/acquisition/modules/waf/utils.go | 10 ++-- pkg/acquisition/modules/waf/waf.go | 72 +++++++++++++++++----------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 55e673ba904..00a88e14691 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -55,11 +55,11 @@ func LogWaapEvent(evt *types.Event) { log.Infof("%s", evt.Waap) } -func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { - log.Infof("TX %v", &tx) +func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { + r.logger.Infof("TX %v", &tx) if tx.IsInterrupted() { - log.Infof("interrupted() = %t", tx.IsInterrupted()) - log.Infof("interrupted.action = %s", tx.Interruption().Action) + r.logger.Infof("interrupted() = %t", tx.IsInterrupted()) + r.logger.Infof("interrupted.action = %s", tx.Interruption().Action) if evt.Meta == nil { evt.Meta = map[string]string{} } @@ -69,7 +69,7 @@ func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *type evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } - //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + //r.logger.Infof("TX %s", spew.Sdump(tx.MatchedRules())) for _, rule := range tx.MatchedRules() { if rule.Message() == "" { continue diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 435d33e2a33..abe9ad1b028 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -17,7 +17,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -38,6 +37,7 @@ type WafRunner struct { outOfBandWaf coraza.WAF UUID string RulesCollections []*waf.WafRulesCollection + logger *log.Entry } type WafSourceConfig struct { @@ -45,6 +45,7 @@ type WafSourceConfig struct { ListenPort int `yaml:"listen_port"` Path string `yaml:"path"` WafRoutines int `yaml:"waf_routines"` + Debug bool `yaml:"debug"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -230,13 +231,30 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { if err != nil { return errors.Wrap(err, "Cannot create WAF") } + wafUUID := uuid.New().String() + wafLogger := &log.Entry{} + if w.config.Debug { + var clog = log.New() + if err := types.ConfigureLogger(clog); err != nil { + log.Fatalf("While creating bucket-specific logger : %s", err) + } + clog.SetLevel(log.DebugLevel) + wafLogger = clog.WithFields(log.Fields{ + "uuid": wafUUID, + }) + } else { + wafLogger = log.WithFields(log.Fields{ + "uuid": wafUUID, + }) + } runner := WafRunner{ outOfBandWaf: outofbandwaf, inBandWaf: inbandwaf, inChan: w.InChan, - UUID: uuid.New().String(), + UUID: wafUUID, RulesCollections: rulesCollections, + logger: wafLogger, } w.WafRunners[nbRoutine] = runner } @@ -310,10 +328,10 @@ func (w *WafSource) Dump() interface{} { return w } -func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) { +func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsedRequest waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) { var in *corazatypes.Interruption if tx.IsRuleEngineOff() { - log.Printf("engine is off") + r.logger.Printf("engine is off") return nil, nil, nil } @@ -329,25 +347,25 @@ func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, //txx := experimental.ToFullInterface(tx) //txx = tx.(experimental.FullTransaction) //txx.RemoveRuleByID(1) - tx.ProcessConnection(r.ClientIP, 0, "", 0) + tx.ProcessConnection(parsedRequest.ClientIP, 0, "", 0) - //tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers - tx.ProcessURI(r.URI, r.Method, r.Proto) //FIXME: get it from the headers + //tx.ProcessURI(parsedRequest.URL.String(), parsedRequest.Method, parsedRequest.Proto) //FIXME: get it from the headers + tx.ProcessURI(parsedRequest.URI, parsedRequest.Method, parsedRequest.Proto) //FIXME: get it from the headers - for k, vr := range r.Headers { + for k, vr := range parsedRequest.Headers { for _, v := range vr { tx.AddRequestHeader(k, v) } } - if r.ClientHost != "" { - tx.AddRequestHeader("Host", r.ClientHost) + if parsedRequest.ClientHost != "" { + tx.AddRequestHeader("Host", parsedRequest.ClientHost) // This connector relies on the host header (now host field) to populate ServerName - tx.SetServerName(r.ClientHost) + tx.SetServerName(parsedRequest.ClientHost) } - if r.TransferEncoding != nil { - tx.AddRequestHeader("Transfer-Encoding", r.TransferEncoding[0]) + if parsedRequest.TransferEncoding != nil { + tx.AddRequestHeader("Transfer-Encoding", parsedRequest.TransferEncoding[0]) } in = tx.ProcessRequestHeaders() @@ -356,7 +374,7 @@ func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, //spew.Dump(tx.MatchedRules()) for _, rule := range tx.MatchedRules() { - log.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) + r.logger.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) if rule.Message() == "" { continue } @@ -367,9 +385,9 @@ func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, return in, tx, nil } - ct := r.Headers.Get("content-type") - if r.Body != nil && len(r.Body) != 0 { - it, _, err := tx.WriteRequestBody(r.Body) + ct := parsedRequest.Headers.Get("content-type") + if parsedRequest.Body != nil && len(parsedRequest.Body) != 0 { + it, _, err := tx.WriteRequestBody(parsedRequest.Body) if err != nil { return nil, nil, errors.Wrap(err, "Cannot read request body") } @@ -405,7 +423,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { for { select { case <-t.Dying(): - log.Infof("Waf Runner is dying") + r.logger.Infof("Waf Runner is dying") return nil case request := <-r.inChan: var evt *types.Event @@ -459,9 +477,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } - in, expTx, err := processReqWithEngine(expTx, request, InBand) + in, expTx, err := r.processReqWithEngine(expTx, request, InBand) request.Tx = expTx - log.Infof("-> %s", spew.Sdump(in)) + //log.Infof("-> %s", spew.Sdump(in)) response := waf.NewResponseRequest(expTx, in, request.UUID, err) @@ -477,7 +495,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { "req": request, }) if err != nil { - log.Errorf("unable to run PreEval filter: %s", err) + r.logger.Errorf("unable to run PreEval filter: %s", err) continue } @@ -487,7 +505,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { continue } default: - log.Errorf("Filter must return a boolean, can't filter") + r.logger.Errorf("Filter must return a boolean, can't filter") continue } } @@ -502,7 +520,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { "CancelEvent": response.CancelEvent, }) if err != nil { - log.Errorf("unable to apply filter: %s", err) + r.logger.Errorf("unable to apply filter: %s", err) continue } } @@ -519,7 +537,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if err != nil { return fmt.Errorf("cannot create event from waap context : %w", err) } - err = AccumulateTxToEvent(expTx, InBand, evt) + err = r.AccumulateTxToEvent(expTx, InBand, evt) if err != nil { return fmt.Errorf("cannot convert transaction to event : %w", err) } @@ -531,9 +549,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // Process outBand outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) expTx = outBandTx.(experimental.FullTransaction) - in, expTx, err = processReqWithEngine(expTx, request, OutOfBand) + in, expTx, err = r.processReqWithEngine(expTx, request, OutOfBand) if err != nil { //things went south - log.Errorf("Error while processing request : %s", err) + r.logger.Errorf("Error while processing request : %s", err) continue } request.Tx = expTx @@ -546,7 +564,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } - err = AccumulateTxToEvent(expTx, InBand, evt) + err = r.AccumulateTxToEvent(expTx, InBand, evt) if err != nil { return fmt.Errorf("cannot convert transaction to event : %w", err) } From ab2c1526270b10dc2ff80380d727207732806a68 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 14:39:57 +0200 Subject: [PATCH 039/263] reduce verbosity --- pkg/acquisition/modules/waf/utils.go | 2 +- pkg/acquisition/modules/waf/waf.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 00a88e14691..9f26f67d5c7 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -52,7 +52,7 @@ func LogWaapEvent(evt *types.Event) { "source": evt.Parsed["source_ip"], "target_uri": evt.Parsed["target_uri"], }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs()) - log.Infof("%s", evt.Waap) + //log.Infof("%s", evt.Waap) } func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index abe9ad1b028..9f4e764bbcc 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -374,7 +374,7 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed //spew.Dump(tx.MatchedRules()) for _, rule := range tx.MatchedRules() { - r.logger.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) + //r.logger.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) if rule.Message() == "" { continue } From 472f40b9d40d0658ac32b2df1ad1555168c39559 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 18:18:24 +0200 Subject: [PATCH 040/263] fix --- pkg/acquisition/modules/waf/waf.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 9f4e764bbcc..61d826c682a 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -293,7 +293,7 @@ func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) err w.logger.Infof("%d waf runner to start", len(w.WafRunners)) for _, runner := range w.WafRunners { - w.logger.Infof("Running waf runner: %s", runner.UUID) + runner := runner runner.outChan = out t.Go(func() error { return runner.Run(t) @@ -420,6 +420,7 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed } func (r *WafRunner) Run(t *tomb.Tomb) error { + r.logger.Infof("Waf Runner ready to process event") for { select { case <-t.Dying(): From 779ea2e2629213abcd20faca3d4ca7ce78ae5f8c Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 19 Jul 2023 18:19:14 +0200 Subject: [PATCH 041/263] fix --- pkg/acquisition/modules/waf/waf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 61d826c682a..3f08edc5465 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -565,7 +565,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } - err = r.AccumulateTxToEvent(expTx, InBand, evt) + err = r.AccumulateTxToEvent(expTx, OutOfBand, evt) if err != nil { return fmt.Errorf("cannot convert transaction to event : %w", err) } From 54fd2e4e70b97103f02ba0aa47b0b4b9cbdb8c2f Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 20 Jul 2023 16:47:07 +0200 Subject: [PATCH 042/263] fixed --- pkg/acquisition/modules/waf/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 9f26f67d5c7..50d2eca7fbc 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -69,7 +69,7 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } - //r.logger.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + for _, rule := range tx.MatchedRules() { if rule.Message() == "" { continue From b33ba277bf369497ce8318bb335fff4f594c3124 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 20 Jul 2023 17:10:01 +0200 Subject: [PATCH 043/263] add flatten to manipulate arrays of arrays --- pkg/exprhelpers/expr_lib.go | 5 +++++ pkg/exprhelpers/helpers.go | 23 ++++++++++++++++++++++- pkg/types/constants.go | 2 +- pkg/types/event.go | 1 + 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index f4e1f4722fd..7880233a314 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -20,6 +20,11 @@ var exprFuncs = []exprCustomFunc{ new(func(string) (*cticlient.SmokeItem, error)), }, }, + { + name: "Flatten", + function: Flatten, + signature: []interface{}{}, + }, { name: "Distance", function: Distance, diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index a5f45c4b076..4122aad846f 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path" + "reflect" "regexp" "strconv" "strings" @@ -170,7 +171,27 @@ func FileInit(fileFolder string, filename string, fileType string) error { return nil } -//Expr helpers +// Expr helpers + +func Flatten(params ...any) (any, error) { + return flatten(nil, reflect.ValueOf(params)), nil +} + +func flatten(args []interface{}, v reflect.Value) []interface{} { + if v.Kind() == reflect.Interface { + v = v.Elem() + } + + if v.Kind() == reflect.Array || v.Kind() == reflect.Slice { + for i := 0; i < v.Len(); i++ { + args = flatten(args, v.Index(i)) + } + } else { + args = append(args, v.Interface()) + } + + return args +} // func Get(arr []string, index int) string { func Get(params ...any) (any, error) { diff --git a/pkg/types/constants.go b/pkg/types/constants.go index fa50b64f367..c82e9a06c6a 100644 --- a/pkg/types/constants.go +++ b/pkg/types/constants.go @@ -9,7 +9,7 @@ const PAPIVersion = "v1" const PAPIPollUrl = "/decisions/stream/poll" const PAPIPermissionsUrl = "/permissions" -const CAPIBaseURL = "https://api.crowdsec.net/" +const CAPIBaseURL = "https://api.dev.crowdsec.net/" const CscliOrigin = "cscli" const CrowdSecOrigin = "crowdsec" diff --git a/pkg/types/event.go b/pkg/types/event.go index a0741c66018..b51d1291670 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -155,6 +155,7 @@ func (w WaapEvent) ByTagRx(rx string) WaapEvent { } for _, rule := range w { for _, tag := range rule["tags"].([]string) { + log.Infof("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) if re.MatchString(tag) { waap = append(waap, rule) break From a326ffbb1e2610ecfbb22c4fc0b583b0a6f7d5e8 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 20 Jul 2023 17:30:58 +0200 Subject: [PATCH 044/263] add distinct --- pkg/exprhelpers/expr_lib.go | 10 ++++++++++ pkg/exprhelpers/helpers.go | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index 7880233a314..1d29539cb81 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -25,6 +25,16 @@ var exprFuncs = []exprCustomFunc{ function: Flatten, signature: []interface{}{}, }, + { + name: "Distinct", + function: Distinct, + signature: []interface{}{}, + }, + { + name: "FlattenDistinct", + function: FlattenDistinct, + signature: []interface{}{}, + }, { name: "Distance", function: Distance, diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 4122aad846f..2309d47b1aa 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -173,6 +173,33 @@ func FileInit(fileFolder string, filename string, fileType string) error { // Expr helpers +func Distinct(params ...any) (any, error) { + + if rt := reflect.TypeOf(params[0]).Kind(); rt != reflect.Slice && rt != reflect.Array { + return nil, nil + } + array := params[0].([]interface{}) + if array == nil { + return nil, nil + } + + var exists map[any]bool = make(map[any]bool) + var ret []interface{} = make([]interface{}, 0) + + for _, val := range array { + if _, ok := exists[val]; !ok { + exists[val] = true + ret = append(ret, val) + } + } + return ret, nil + +} + +func FlattenDistinct(params ...any) (any, error) { + return Distinct(flatten(nil, reflect.ValueOf(params))) +} + func Flatten(params ...any) (any, error) { return flatten(nil, reflect.ValueOf(params)), nil } From c17b103f06e78585f1e3fb88c5edb403449785c7 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 25 Jul 2023 15:24:36 +0200 Subject: [PATCH 045/263] take method from header --- pkg/waf/request.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/waf/request.go b/pkg/waf/request.go index a5a0c5ffdf0..1cf28a9f322 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -11,6 +11,13 @@ import ( "github.com/google/uuid" ) +const ( + URIHeaderName = "X-Crowdsec-Waf-Uri" + VerbHeaderName = "X-Crowdsec-Waf-Verb" + HostHeaderName = "X-Crowdsec-Waf-Host" + IPHeaderName = "X-Crowdsec-Waf-Ip" +) + type ResponseRequest struct { UUID string Tx corazatypes.Transaction @@ -77,16 +84,19 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { } // the real source of the request is set in 'x-client-ip' - clientIP := r.Header.Get("X-Client-Ip") + clientIP := r.Header.Get(IPHeaderName) // the real target Host of the request is set in 'x-client-host' - clientHost := r.Header.Get("X-Client-Host") + clientHost := r.Header.Get(HostHeaderName) // the real URI of the request is set in 'x-client-uri' - clientURI := r.Header.Get("X-Client-Uri") + clientURI := r.Header.Get(URIHeaderName) + // the real VERB of the request is set in 'x-client-uri' + clientMethod := r.Header.Get(VerbHeaderName) // delete those headers before coraza process the request - delete(r.Header, "x-client-ip") - delete(r.Header, "x-client-host") - delete(r.Header, "x-client-uri") + delete(r.Header, IPHeaderName) + delete(r.Header, HostHeaderName) + delete(r.Header, URIHeaderName) + delete(r.Header, VerbHeaderName) return ParsedRequest{ RemoteAddr: r.RemoteAddr, @@ -94,10 +104,10 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { ClientHost: clientHost, ClientIP: clientIP, URI: clientURI, + Method: clientMethod, Host: r.Host, Headers: r.Header, URL: r.URL, - Method: r.Method, Proto: r.Proto, Body: body, TransferEncoding: r.TransferEncoding, From 4993758b36e3a4a066b46fff96b68874a6dcf62f Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 26 Jul 2023 12:47:16 +0200 Subject: [PATCH 046/263] handle missing headers --- pkg/acquisition/modules/waf/waf.go | 7 ++++--- pkg/waf/request.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 3f08edc5465..355f7aeff98 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -591,7 +591,7 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { parsedRequest, err := waf.NewParsedRequestFromRequest(r) if err != nil { log.Errorf("%s", err) - rw.WriteHeader(http.StatusForbidden) + rw.WriteHeader(http.StatusInternalServerError) return } w.InChan <- parsedRequest @@ -600,7 +600,7 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { if message.Err != nil { log.Errorf("Error while processing InBAND: %s", err) - rw.WriteHeader(http.StatusOK) + rw.WriteHeader(http.StatusInternalServerError) return } @@ -622,7 +622,8 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusOK) body, err := json.Marshal(BodyResponse{Action: "allow"}) if err != nil { - log.Errorf("unable to build response: %s", err) + log.Errorf("unable to marshal response: %s", err) + rw.WriteHeader(http.StatusInternalServerError) } else { rw.Write(body) } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 1cf28a9f322..42cd9d6d55d 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -85,12 +85,24 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { // the real source of the request is set in 'x-client-ip' clientIP := r.Header.Get(IPHeaderName) + if clientIP == "" { + return ParsedRequest{}, fmt.Errorf("Missing '%s' header", IPHeaderName) + } // the real target Host of the request is set in 'x-client-host' clientHost := r.Header.Get(HostHeaderName) + if clientHost == "" { + return ParsedRequest{}, fmt.Errorf("Missing '%s' header", HostHeaderName) + } // the real URI of the request is set in 'x-client-uri' clientURI := r.Header.Get(URIHeaderName) + if clientURI == "" { + return ParsedRequest{}, fmt.Errorf("Missing '%s' header", URIHeaderName) + } // the real VERB of the request is set in 'x-client-uri' clientMethod := r.Header.Get(VerbHeaderName) + if clientMethod == "" { + return ParsedRequest{}, fmt.Errorf("Missing '%s' header", VerbHeaderName) + } // delete those headers before coraza process the request delete(r.Header, IPHeaderName) From 01ced8fb996b8bcb9743890adb96e68cc768709c Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 24 Jul 2023 09:13:26 +0200 Subject: [PATCH 047/263] merge --- go.mod | 2 ++ pkg/acquisition/modules/waf/utils.go | 15 +++++++---- pkg/acquisition/modules/waf/waf.go | 38 +++++++++++++++++++++++----- pkg/waf/waf.go | 3 +-- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 9fedf14a8d2..5b6ba5ed071 100644 --- a/go.mod +++ b/go.mod @@ -204,3 +204,5 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 + +replace github.com/crowdsecurity/coraza/v3 => /home/seb/taff/crowdsec/git/crowdsec-org/coraza diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 50d2eca7fbc..9c24623e0a9 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -8,7 +8,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" ) func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { @@ -47,16 +46,17 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { } func LogWaapEvent(evt *types.Event) { - log.WithFields(log.Fields{ + /*log.WithFields(log.Fields{ "module": "waf", "source": evt.Parsed["source_ip"], "target_uri": evt.Parsed["target_uri"], - }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs()) + }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs())*/ //log.Infof("%s", evt.Waap) } func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { - r.logger.Infof("TX %v", &tx) + + //log.Infof("tx addr: %p", tx) if tx.IsInterrupted() { r.logger.Infof("interrupted() = %t", tx.IsInterrupted()) r.logger.Infof("interrupted.action = %s", tx.Interruption().Action) @@ -66,10 +66,15 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Parsed["interrupted"] = "true" evt.Parsed["action"] = tx.Interruption().Action + //log.Infof("action: %s", tx.Interruption().Action) + evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } - + r.logger.Infof("variables addr in AccumulateTxToEvent: %p", tx.Variables()) + //log.Infof("variables: %s", spew.Sdump(tx.Variables())) + //log.Infof("tx variables: %+v", tx.Collection(variables.TX)) + //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) for _, rule := range tx.MatchedRules() { if rule.Message() == "" { continue diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 355f7aeff98..06e0e068313 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -11,8 +11,10 @@ import ( "github.com/antonmedv/expr" "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/collection" "github.com/crowdsecurity/coraza/v3/experimental" corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/coraza/v3/types/variables" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" @@ -206,6 +208,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.InChan = make(chan waf.ParsedRequest) + w.logger.Infof("w.InChan creation: %p", w.InChan) w.WafRunners = make([]WafRunner, w.config.WafRoutines) for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) @@ -259,10 +262,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.WafRunners[nbRoutine] = runner } - w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) - if err != nil { - return errors.Wrap(err, "Cannot create WAF") - } + w.logger.Infof("Created %d waf runners", len(w.WafRunners)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) @@ -296,6 +296,7 @@ func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) err runner := runner runner.outChan = out t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/waf/live/runner") return runner.Run(t) }) } @@ -370,15 +371,26 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed in = tx.ProcessRequestHeaders() + for _, v := range tx.Collection(variables.TX).FindAll() { + log.Infof("tx variable: %s | %s", v.Key(), v.Value()) + } + + tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { + log.Infof("Collection: %s", col.Name()) + log.Infof("Variable: %s", v.Name()) + //collect := tx.Collection(col) + return true + }) + //spew.Dump(in) //spew.Dump(tx.MatchedRules()) - for _, rule := range tx.MatchedRules() { - //r.logger.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) + /*for _, rule := range tx.MatchedRules() { + log.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) if rule.Message() == "" { continue } - } + }*/ //if we're inband, we should stop here, but for outofband go to the end if in != nil && wafType == InBand { @@ -393,6 +405,7 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed } if it != nil { + //log.Infof("blocking rule id %d", in.RuleID) return it, nil, nil } // from https://github.com/corazawaf/coraza/blob/main/internal/corazawaf/transaction.go#L419 @@ -413,6 +426,8 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed return nil, nil, errors.Wrap(err, "Cannot process request body") } if in != nil && wafType == InBand { + //log.Infof("blocking rule id %d", in.RuleID) + return in, tx, nil } @@ -427,6 +442,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { r.logger.Infof("Waf Runner is dying") return nil case request := <-r.inChan: + r.logger.Infof("Requests handled by runner %s", r.UUID) var evt *types.Event WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() //measure the time spent in the WAF @@ -436,6 +452,8 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // we use this internal transaction for the expr helpers tx := waf.NewTransaction(expTx) + //r.logger.Infof("Processing request %s | tx: %p", request.UUID, tx) + //Run the pre_eval hooks for _, rules := range r.RulesCollections { if len(rules.CompiledPreEval) == 0 { @@ -482,6 +500,12 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { request.Tx = expTx //log.Infof("-> %s", spew.Sdump(in)) + //log.Infof("tx variables: %+v", expTx.Collection(variables.TX)) + + //foo := expTx.(plugintypes.TransactionState) + + //log.Infof("from tstate: %+v", foo.Variables().TX().FindAll()) + response := waf.NewResponseRequest(expTx, in, request.UUID, err) // run the on_match hooks diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index d2d4d0dc8eb..9e674f01132 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -11,7 +11,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -104,7 +103,7 @@ func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { continue } - spew.Dump(wafConfig) + //spew.Dump(wafConfig) collection := &WafRulesCollection{} From 792961d757e87cbe032a4bd106bb1025b784d580 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 24 Jul 2023 14:50:08 +0200 Subject: [PATCH 048/263] wip --- pkg/acquisition/modules/waf/utils.go | 10 ++++ pkg/acquisition/modules/waf/waf.go | 19 ++------ pkg/types/event.go | 69 +++++++++++++++++++--------- 3 files changed, 61 insertions(+), 37 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 9c24623e0a9..49c5bf71f95 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -4,7 +4,9 @@ import ( "fmt" "time" + "github.com/crowdsecurity/coraza/v3/collection" "github.com/crowdsecurity/coraza/v3/experimental" + "github.com/crowdsecurity/coraza/v3/types/variables" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" @@ -71,6 +73,14 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Meta["waap_interrupted"] = "1" evt.Meta["waap_action"] = tx.Interruption().Action } + + tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { + for _, variable := range col.FindAll() { + r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + } + return true + }) + r.logger.Infof("variables addr in AccumulateTxToEvent: %p", tx.Variables()) //log.Infof("variables: %s", spew.Sdump(tx.Variables())) //log.Infof("tx variables: %+v", tx.Collection(variables.TX)) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 06e0e068313..749576314ec 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -11,10 +11,8 @@ import ( "github.com/antonmedv/expr" "github.com/crowdsecurity/coraza/v3" - "github.com/crowdsecurity/coraza/v3/collection" "github.com/crowdsecurity/coraza/v3/experimental" corazatypes "github.com/crowdsecurity/coraza/v3/types" - "github.com/crowdsecurity/coraza/v3/types/variables" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" @@ -338,7 +336,8 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed defer func() { tx.ProcessLogging() - tx.Close() + //Dont close the transaction here: we still need access to the variables afterwards + //tx.Close() }() //this method is not exported by coraza, so we have to do it ourselves. @@ -371,17 +370,6 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed in = tx.ProcessRequestHeaders() - for _, v := range tx.Collection(variables.TX).FindAll() { - log.Infof("tx variable: %s | %s", v.Key(), v.Value()) - } - - tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { - log.Infof("Collection: %s", col.Name()) - log.Infof("Variable: %s", v.Name()) - //collect := tx.Collection(col) - return true - }) - //spew.Dump(in) //spew.Dump(tx.MatchedRules()) @@ -569,6 +557,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { LogWaapEvent(evt) r.outChan <- *evt } + expTx.Close() outBandStart := time.Now() // Process outBand @@ -595,8 +584,8 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } LogWaapEvent(evt) r.outChan <- *evt - } + expTx.Close() //measure the full time spent in the WAF totalElapsed := time.Since(startParsing) WafGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(totalElapsed.Seconds()) diff --git a/pkg/types/event.go b/pkg/types/event.go index b51d1291670..0405c3466ee 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -3,6 +3,7 @@ package types import ( "fmt" "regexp" + "strings" "time" log "github.com/sirupsen/logrus" @@ -27,8 +28,12 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 */ -type WaapEvent []map[string]interface{} +type MatchedRules []map[string]interface{} +type WaapEvent struct { + MatchedRules + Vars map[string]string +} type Field string func (f Field) String() string { @@ -49,8 +54,28 @@ const ( Kind Field = "kind" ) +func (w WaapEvent) GetVar(varName string) string { + if w.Vars == nil { + return "" + } + parsed := strings.Split(varName, ".") + if len(parsed) == 1 { + //no subkey + return w.Vars[varName] + } else if len(parsed) == 2 { + //subkey + if w.Vars[parsed[0]] == "" { + return "" + } + return w.Vars[parsed[0]][parsed[1]] + } + log.Warningf("invalid variable name %s", varName) + return "" + +} + // getters -func (w WaapEvent) GetField(field Field) []interface{} { +func (w MatchedRules) GetField(field Field) []interface{} { ret := make([]interface{}, 0) for _, rule := range w { ret = append(ret, rule[field.String()]) @@ -58,21 +83,21 @@ func (w WaapEvent) GetField(field Field) []interface{} { return ret } -func (w WaapEvent) GetURI() string { +func (w MatchedRules) GetURI() string { for _, rule := range w { return rule["uri"].(string) } return "" } -func (w WaapEvent) GetMethod() string { +func (w MatchedRules) GetMethod() string { for _, rule := range w { return rule["method"].(string) } return "" } -func (w WaapEvent) GetRuleIDs() []int { +func (w MatchedRules) GetRuleIDs() []int { ret := make([]int, 0) for _, rule := range w { ret = append(ret, rule["id"].(int)) @@ -80,7 +105,7 @@ func (w WaapEvent) GetRuleIDs() []int { return ret } -func (w WaapEvent) Kinds() []string { +func (w MatchedRules) Kinds() []string { ret := make([]string, 0) for _, rule := range w { exists := false @@ -98,8 +123,8 @@ func (w WaapEvent) Kinds() []string { } // filters -func (w WaapEvent) ByID(id int) WaapEvent { - waap := WaapEvent{} +func (w MatchedRules) ByID(id int) MatchedRules { + waap := MatchedRules{} for _, rule := range w { if rule["id"] == id { @@ -109,8 +134,8 @@ func (w WaapEvent) ByID(id int) WaapEvent { return waap } -func (w WaapEvent) ByKind(kind string) WaapEvent { - waap := WaapEvent{} +func (w MatchedRules) ByKind(kind string) MatchedRules { + waap := MatchedRules{} for _, rule := range w { if rule["kind"] == kind { waap = append(waap, rule) @@ -119,8 +144,8 @@ func (w WaapEvent) ByKind(kind string) WaapEvent { return waap } -func (w WaapEvent) ByTags(match []string) WaapEvent { - waap := WaapEvent{} +func (w MatchedRules) ByTags(match []string) MatchedRules { + waap := MatchedRules{} for _, rule := range w { for _, tag := range rule["tags"].([]string) { for _, match_tag := range match { @@ -134,8 +159,8 @@ func (w WaapEvent) ByTags(match []string) WaapEvent { return waap } -func (w WaapEvent) ByTag(match string) WaapEvent { - waap := WaapEvent{} +func (w MatchedRules) ByTag(match string) MatchedRules { + waap := MatchedRules{} for _, rule := range w { for _, tag := range rule["tags"].([]string) { if tag == match { @@ -147,8 +172,8 @@ func (w WaapEvent) ByTag(match string) WaapEvent { return waap } -func (w WaapEvent) ByTagRx(rx string) WaapEvent { - waap := WaapEvent{} +func (w MatchedRules) ByTagRx(rx string) MatchedRules { + waap := MatchedRules{} re := regexp.MustCompile(rx) if re == nil { return waap @@ -165,9 +190,9 @@ func (w WaapEvent) ByTagRx(rx string) WaapEvent { return waap } -func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent { +func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules { log.Infof("%s", w) - wap := WaapEvent{} + wap := MatchedRules{} for _, rule := range w { if rule["disruptive"] == is { wap = append(wap, rule) @@ -178,8 +203,8 @@ func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent { return wap } -func (w WaapEvent) BySeverity(severity string) WaapEvent { - wap := WaapEvent{} +func (w MatchedRules) BySeverity(severity string) MatchedRules { + wap := MatchedRules{} for _, rule := range w { if rule["severity"] == severity { wap = append(wap, rule) @@ -189,8 +214,8 @@ func (w WaapEvent) BySeverity(severity string) WaapEvent { return wap } -func (w WaapEvent) ByAccuracy(accuracy string) WaapEvent { - wap := WaapEvent{} +func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules { + wap := MatchedRules{} for _, rule := range w { if rule["accuracy"] == accuracy { wap = append(wap, rule) From f7e098047f25b7fc079edf20df15746fb4cd85b5 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 24 Jul 2023 15:22:30 +0200 Subject: [PATCH 049/263] waf_rules -> waf-rules --- pkg/cwhub/cwhub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 0e508d9f1d5..5e9c0ddf040 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -78,7 +78,7 @@ type Item struct { PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` - WafRules []string `yaml:"waf_rules,omitempty" json:"waf_rules,omitempty"` + WafRules []string `yaml:"waf-rules,omitempty" json:"waf-rules,omitempty"` } func (i *Item) toHubStatus() ItemHubStatus { From 2f5a6fbb4f537bf78f0c4cfca7633a98a98d4e82 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 24 Jul 2023 15:22:44 +0200 Subject: [PATCH 050/263] wip --- pkg/acquisition/modules/waf/utils.go | 4 ++-- pkg/types/event.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 49c5bf71f95..0e13abaf373 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -42,7 +42,7 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { Src: "waf", Raw: "dummy-waf-data", //we discard empty Line.Raw items :) } - evt.Waap = []map[string]interface{}{} + evt.Waap = types.WaapEvent{} return evt, nil } @@ -106,7 +106,7 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st "msg": rule.Message(), "severity": rule.Rule().Severity().String(), } - evt.Waap = append(evt.Waap, corazaRule) + evt.Waap.MatchedRules = append(evt.Waap.MatchedRules, corazaRule) } return nil diff --git a/pkg/types/event.go b/pkg/types/event.go index 0405c3466ee..0709e06a706 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -67,7 +67,7 @@ func (w WaapEvent) GetVar(varName string) string { if w.Vars[parsed[0]] == "" { return "" } - return w.Vars[parsed[0]][parsed[1]] + //return w.Vars[parsed[0]][parsed[1]] } log.Warningf("invalid variable name %s", varName) return "" From dd5e38a2c53afad552fd1d60a59f61f39748ca6a Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 27 Jul 2023 10:01:56 +0200 Subject: [PATCH 051/263] expose internal coraza vars in evt.Waap --- pkg/acquisition/modules/waf/utils.go | 14 ++++++++++++++ pkg/types/event.go | 15 +++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 0e13abaf373..e91607f608b 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -74,8 +74,22 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Meta["waap_action"] = tx.Interruption().Action } + if evt.Waap.Vars == nil { + evt.Waap.Vars = map[string]string{} + } + tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { for _, variable := range col.FindAll() { + key := "" + if variable.Key() == "" { + key = variable.Variable().Name() + } else { + key = variable.Variable().Name() + "." + variable.Key() + } + if variable.Value() == "" { + continue + } + evt.Waap.Vars[key] = variable.Value() r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } return true diff --git a/pkg/types/event.go b/pkg/types/event.go index 0709e06a706..d3b21c35f33 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -3,7 +3,6 @@ package types import ( "fmt" "regexp" - "strings" "time" log "github.com/sirupsen/logrus" @@ -58,18 +57,10 @@ func (w WaapEvent) GetVar(varName string) string { if w.Vars == nil { return "" } - parsed := strings.Split(varName, ".") - if len(parsed) == 1 { - //no subkey - return w.Vars[varName] - } else if len(parsed) == 2 { - //subkey - if w.Vars[parsed[0]] == "" { - return "" - } - //return w.Vars[parsed[0]][parsed[1]] + if val, ok := w.Vars[varName]; ok { + return val } - log.Warningf("invalid variable name %s", varName) + log.Infof("var %s not found", varName, w.Vars) return "" } From c41386056acff6f08cc567a97c7cbc7ea090aa3b Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 27 Jul 2023 10:04:24 +0200 Subject: [PATCH 052/263] remove local replace --- go.mod | 16 +++++++--------- go.sum | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 5b6ba5ed071..272fbc37091 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.11.0 golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 @@ -71,7 +71,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/cespare/xxhash/v2 v2.1.2 github.com/coreos/go-systemd/v22 v22.5.0 - github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5 + github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd github.com/goccy/go-yaml v1.9.7 github.com/gofrs/uuid v4.0.0+incompatible @@ -87,7 +87,7 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v0.2.1 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sys v0.9.0 + golang.org/x/sys v0.10.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -162,7 +162,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect - github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -185,10 +185,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -204,5 +204,3 @@ require ( ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 - -replace github.com/crowdsecurity/coraza/v3 => /home/seb/taff/crowdsec/git/crowdsec-org/coraza diff --git a/go.sum b/go.sum index 19fcd83317e..6b64c39cfc9 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5 h1:zpU5j3EOvH/Hxs3trWwsXp7Vr1T0YP0MIdp89RvVlY8= github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= +github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f h1:7MgSs0ryJrdGV0f17xBQJPfdD8oo/mqcIQjpqdPo2aA= +github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= @@ -762,6 +764,8 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e h1:POJco99aNgosh92lGqmx7L1ei+kCymivB/419SD15PQ= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -1023,6 +1027,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1111,6 +1117,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1214,12 +1222,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1231,6 +1243,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From a7cd86f725062feb979c75ec16cf866b5dda2a6d Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 31 Jul 2023 12:15:04 +0200 Subject: [PATCH 053/263] allow to select what variables shouldd be tracked --- pkg/acquisition/modules/waf/utils.go | 29 +++++++++++++++++++-- pkg/acquisition/modules/waf/waf.go | 39 +++++++++++++++------------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index e91607f608b..1831bc6e16d 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -2,6 +2,7 @@ package wafacquisition import ( "fmt" + "regexp" "time" "github.com/crowdsecurity/coraza/v3/collection" @@ -56,6 +57,13 @@ func LogWaapEvent(evt *types.Event) { //log.Infof("%s", evt.Waap) } +/* + how to configure variables to be kept: + 1) full collection : tx.* + 2) subvariables : tx.a* + +*/ + func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { //log.Infof("tx addr: %p", tx) @@ -78,6 +86,12 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Waap.Vars = map[string]string{} } + // collectionsToKeep := []string{ + // "toto", + // "TX.allowed_methods", + // "TX.*_score", + // } + tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { for _, variable := range col.FindAll() { key := "" @@ -89,8 +103,19 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st if variable.Value() == "" { continue } - evt.Waap.Vars[key] = variable.Value() - r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + for _, collectionToKeep := range r.VariablesTracking { + match, err := regexp.MatchString("(?i)"+collectionToKeep, key) + if err != nil { + r.logger.Warningf("error matching %s with %s: %s", key, collectionToKeep, err) + continue + } + if match { + evt.Waap.Vars[key] = variable.Value() + r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + } else { + r.logger.Infof("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) + } + } } return true }) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 749576314ec..3bc1e0a8352 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -31,21 +31,23 @@ const ( ) type WafRunner struct { - outChan chan types.Event - inChan chan waf.ParsedRequest - inBandWaf coraza.WAF - outOfBandWaf coraza.WAF - UUID string - RulesCollections []*waf.WafRulesCollection - logger *log.Entry + outChan chan types.Event + inChan chan waf.ParsedRequest + inBandWaf coraza.WAF + outOfBandWaf coraza.WAF + UUID string + RulesCollections []*waf.WafRulesCollection + logger *log.Entry + VariablesTracking []string } type WafSourceConfig struct { - ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` - Path string `yaml:"path"` - WafRoutines int `yaml:"waf_routines"` - Debug bool `yaml:"debug"` + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + Path string `yaml:"path"` + WafRoutines int `yaml:"waf_routines"` + Debug bool `yaml:"debug"` + VariablesTracking []string `yaml:"variables_tracking"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -250,12 +252,13 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { } runner := WafRunner{ - outOfBandWaf: outofbandwaf, - inBandWaf: inbandwaf, - inChan: w.InChan, - UUID: wafUUID, - RulesCollections: rulesCollections, - logger: wafLogger, + outOfBandWaf: outofbandwaf, + inBandWaf: inbandwaf, + inChan: w.InChan, + UUID: wafUUID, + RulesCollections: rulesCollections, + logger: wafLogger, + VariablesTracking: w.config.VariablesTracking, } w.WafRunners[nbRoutine] = runner } From e4e2bb550413e71cb10532243aab3028ad8c1076 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 31 Jul 2023 14:45:21 +0200 Subject: [PATCH 054/263] switch to properly compiled regexp to be able to bail out early --- pkg/acquisition/modules/waf/utils.go | 9 ++------- pkg/acquisition/modules/waf/waf.go | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 1831bc6e16d..03a17fef904 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -2,7 +2,6 @@ package wafacquisition import ( "fmt" - "regexp" "time" "github.com/crowdsecurity/coraza/v3/collection" @@ -104,16 +103,12 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st continue } for _, collectionToKeep := range r.VariablesTracking { - match, err := regexp.MatchString("(?i)"+collectionToKeep, key) - if err != nil { - r.logger.Warningf("error matching %s with %s: %s", key, collectionToKeep, err) - continue - } + match := collectionToKeep.MatchString(key) if match { evt.Waap.Vars[key] = variable.Value() r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } else { - r.logger.Infof("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) + r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) } } } diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 3bc1e0a8352..f6a620e7fec 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "os" + "regexp" "strings" "time" @@ -38,7 +39,7 @@ type WafRunner struct { UUID string RulesCollections []*waf.WafRulesCollection logger *log.Entry - VariablesTracking []string + VariablesTracking []*regexp.Regexp } type WafSourceConfig struct { @@ -251,6 +252,16 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { }) } + var compiledVariableRules []*regexp.Regexp + + for _, variable := range w.config.VariablesTracking { + compiledVariableRule, err := regexp.Compile(variable) + if err != nil { + return fmt.Errorf("cannot compile variable regexp %s: %w", variable, err) + } + compiledVariableRules = append(compiledVariableRules, compiledVariableRule) + } + runner := WafRunner{ outOfBandWaf: outofbandwaf, inBandWaf: inbandwaf, @@ -258,7 +269,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { UUID: wafUUID, RulesCollections: rulesCollections, logger: wafLogger, - VariablesTracking: w.config.VariablesTracking, + VariablesTracking: compiledVariableRules, } w.WafRunners[nbRoutine] = runner } From 4a38cb5bbbb0e2f47119e40ed3fec485fcc2a69c Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 31 Jul 2023 14:47:48 +0200 Subject: [PATCH 055/263] logging --- pkg/acquisition/modules/waf/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 03a17fef904..f3211459230 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -67,8 +67,8 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st //log.Infof("tx addr: %p", tx) if tx.IsInterrupted() { - r.logger.Infof("interrupted() = %t", tx.IsInterrupted()) - r.logger.Infof("interrupted.action = %s", tx.Interruption().Action) + //r.logger.Infof("interrupted() = %t", tx.IsInterrupted()) + //r.logger.Infof("interrupted.action = %s", tx.Interruption().Action) if evt.Meta == nil { evt.Meta = map[string]string{} } @@ -106,7 +106,7 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st match := collectionToKeep.MatchString(key) if match { evt.Waap.Vars[key] = variable.Value() - r.logger.Infof("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } else { r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) } From fc8a0ee9d498aff319d45b863fb44736a0d8bc24 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 15:06:42 +0200 Subject: [PATCH 056/263] update --- pkg/acquisition/modules/waf/waf.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index f6a620e7fec..7c513008f77 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -399,7 +399,6 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed return in, tx, nil } - ct := parsedRequest.Headers.Get("content-type") if parsedRequest.Body != nil && len(parsedRequest.Body) != 0 { it, _, err := tx.WriteRequestBody(parsedRequest.Body) if err != nil { @@ -410,19 +409,8 @@ func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsed //log.Infof("blocking rule id %d", in.RuleID) return it, nil, nil } - // from https://github.com/corazawaf/coraza/blob/main/internal/corazawaf/transaction.go#L419 - // urlencoded cannot end with CRLF - if ct != "application/x-www-form-urlencoded" { - it, _, err := tx.WriteRequestBody([]byte{'\r', '\n'}) - if err != nil { - return nil, nil, fmt.Errorf("cannot write to request body to buffer: %s", err.Error()) - } - - if it != nil { - return it, nil, nil - } - } } + in, err := tx.ProcessRequestBody() if err != nil { return nil, nil, errors.Wrap(err, "Cannot process request body") From dd83bdea6b9c6920c97e9df6503dc79e1ede89cb Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 31 Jul 2023 17:00:06 +0200 Subject: [PATCH 057/263] revert previous bad merge --- .github/workflows/bats-hub.yml | 28 +- .github/workflows/bats-mysql.yml | 28 +- .github/workflows/bats-postgres.yml | 28 +- .github/workflows/bats-sqlite-coverage.yml | 28 +- .github/workflows/cache-cleanup.yaml | 35 + .github/workflows/ci-windows-build-msi.yml | 23 +- .github/workflows/docker-tests.yml | 12 - .github/workflows/go-tests-windows.yml | 23 +- .github/workflows/go-tests.yml | 31 +- .github/workflows/release_publish-package.yml | 27 +- .gitignore | 9 +- .golangci.yml | 2 +- Dockerfile | 20 +- Dockerfile.debian | 30 +- Makefile | 99 +- azure-pipelines.yml | 4 +- cmd/crowdsec-cli/alerts.go | 8 +- cmd/crowdsec-cli/capi.go | 19 +- cmd/crowdsec-cli/config_backup.go | 42 +- cmd/crowdsec-cli/config_restore.go | 24 +- cmd/crowdsec-cli/copyfile.go | 73 ++ cmd/crowdsec-cli/decisions.go | 245 +---- cmd/crowdsec-cli/decisions_import.go | 272 +++++ cmd/crowdsec-cli/lapi.go | 6 +- cmd/crowdsec-cli/machines.go | 33 +- cmd/crowdsec-cli/machines_table.go | 6 +- cmd/crowdsec-cli/notifications.go | 58 +- cmd/crowdsec-cli/scenarios.go | 3 +- cmd/crowdsec-cli/support.go | 11 +- cmd/crowdsec-cli/utils.go | 6 +- cmd/crowdsec/api.go | 11 +- cmd/crowdsec/crowdsec.go | 19 +- cmd/crowdsec/main.go | 46 +- cmd/crowdsec/output.go | 15 +- cmd/crowdsec/pour.go | 4 +- cmd/crowdsec/run_in_svc_windows.go | 13 +- cmd/crowdsec/serve.go | 35 +- cmd/crowdsec/win_service.go | 4 +- debian/control | 4 +- debian/rules | 12 +- debian/templates | 2 +- docker/docker_start.sh | 2 +- docker/test/tests/test_hub_collections.py | 32 +- go.mod | 70 +- go.sum | 69 +- mk/__gmsl | 969 ++++++++++++++++++ mk/gmsl | 85 ++ mk/gmsl.html | 733 +++++++++++++ mk/platform/unix_common.mk | 8 +- mk/platform/windows.mk | 8 +- pkg/acquisition/acquisition.go | 46 +- pkg/acquisition/acquisition_test.go | 2 +- .../modules/cloudwatch/cloudwatch.go | 16 +- pkg/acquisition/modules/docker/docker.go | 7 +- pkg/acquisition/modules/file/file.go | 22 +- pkg/acquisition/modules/file/file_test.go | 4 +- .../modules/journalctl/journalctl.go | 3 +- pkg/acquisition/modules/kafka/kafka.go | 11 +- pkg/acquisition/modules/kafka/kafka_test.go | 13 +- pkg/acquisition/modules/kinesis/kinesis.go | 37 +- .../modules/kubernetesaudit/k8s_audit.go | 14 +- pkg/acquisition/modules/s3/s3.go | 13 +- .../syslog/internal/server/syslogserver.go | 9 +- pkg/acquisition/modules/syslog/syslog.go | 5 +- pkg/apiclient/alerts_service.go | 10 +- pkg/apiclient/alerts_service_test.go | 7 +- pkg/apiclient/auth.go | 26 +- pkg/apiclient/auth_service.go | 2 - pkg/apiclient/auth_service_test.go | 5 +- pkg/apiclient/client.go | 7 +- pkg/apiclient/client_http_test.go | 4 +- pkg/apiclient/client_test.go | 3 +- pkg/apiclient/decisions_service.go | 7 +- pkg/apiclient/decisions_service_test.go | 7 +- pkg/apiclient/decisions_sync_service.go | 8 +- pkg/apiclient/signal.go | 5 +- pkg/apiserver/apic.go | 84 +- pkg/apiserver/apic_metrics.go | 145 +++ pkg/apiserver/apic_metrics_test.go | 101 ++ pkg/apiserver/apic_test.go | 84 -- pkg/apiserver/apiserver.go | 28 +- pkg/apiserver/controllers/v1/controller.go | 6 +- pkg/apiserver/middlewares/v1/api_key.go | 10 +- pkg/apiserver/middlewares/v1/jwt.go | 2 +- pkg/apiserver/middlewares/v1/tls_auth.go | 7 +- pkg/csconfig/common.go | 3 +- pkg/csconfig/config_paths.go | 4 +- pkg/csconfig/profiles.go | 4 +- pkg/csplugin/broker.go | 12 +- pkg/csplugin/broker_suite_test.go | 18 +- pkg/csplugin/broker_test.go | 19 +- pkg/csplugin/broker_win_test.go | 2 +- pkg/csplugin/listfiles.go | 3 +- pkg/csplugin/listfiles_test.go | 10 +- pkg/csplugin/notifier.go | 3 +- pkg/csplugin/utils.go | 11 +- pkg/csplugin/utils_windows.go | 35 +- pkg/csplugin/watcher.go | 3 +- pkg/csplugin/watcher_test.go | 17 +- pkg/csprofiles/csprofiles.go | 9 +- pkg/csprofiles/csprofiles_test.go | 7 +- pkg/cticlient/example/fire.go | 3 + pkg/cticlient/types.go | 2 +- pkg/cwhub/cwhub.go | 13 +- pkg/cwhub/cwhub_test.go | 18 +- pkg/{types => cwhub}/dataset.go | 20 +- pkg/{types => cwhub}/dataset_test.go | 4 +- pkg/cwhub/download.go | 5 +- pkg/cwhub/helpers.go | 14 +- pkg/cwhub/helpers_test.go | 6 +- pkg/cwhub/loader.go | 7 +- pkg/database/alerts.go | 35 +- pkg/database/bouncers.go | 3 +- pkg/database/config.go | 3 +- pkg/database/decisions.go | 6 +- pkg/database/machines.go | 4 +- pkg/exprhelpers/jsonextract.go | 4 +- pkg/hubtest/hubtest.go | 3 +- pkg/hubtest/hubtest_item.go | 3 +- pkg/hubtest/parser_assert.go | 8 +- pkg/hubtest/scenario_assert.go | 10 +- pkg/leakybucket/README.md | 120 ++- pkg/leakybucket/bayesian.go | 163 +++ pkg/leakybucket/bucket.go | 4 + pkg/leakybucket/manager_load.go | 29 + pkg/leakybucket/manager_load_test.go | 22 + pkg/leakybucket/overflows.go | 19 +- .../guillotine-bayesian-bucket/bucket.yaml | 21 + .../guillotine-bayesian-bucket/scenarios.yaml | 1 + .../guillotine-bayesian-bucket/test.json | 50 + .../multiple-bayesian-bucket/bucket.yaml | 21 + .../multiple-bayesian-bucket/scenarios.yaml | 1 + .../tests/multiple-bayesian-bucket/test.json | 64 ++ .../tests/simple-bayesian-bucket/bucket.yaml | 19 + .../simple-bayesian-bucket/scenarios.yaml | 1 + .../tests/simple-bayesian-bucket/test.json | 50 + pkg/metabase/api.go | 10 +- pkg/metabase/database.go | 9 +- pkg/metabase/metabase.go | 40 +- pkg/parser/enrich.go | 5 +- pkg/parser/enrich_date.go | 3 +- pkg/parser/enrich_dns.go | 4 +- pkg/parser/enrich_geoip.go | 6 +- pkg/parser/enrich_unmarshal.go | 3 +- pkg/{types => parser}/grok_pattern.go | 3 +- pkg/parser/node.go | 139 ++- pkg/parser/node_test.go | 18 +- pkg/parser/parsing_test.go | 12 +- pkg/parser/runtime.go | 32 +- pkg/parser/stage.go | 17 +- pkg/parser/unix_parser.go | 11 +- pkg/parser/whitelist.go | 1 + pkg/setup/detect.go | 7 +- pkg/setup/detect_test.go | 13 +- pkg/types/datasource.go | 16 + pkg/types/ip.go | 13 +- pkg/types/utils.go | 98 +- plugins/notifications/dummy/Makefile | 5 - plugins/notifications/email/Makefile | 5 - plugins/notifications/email/go.mod | 6 +- plugins/notifications/email/go.sum | 18 +- plugins/notifications/http/Makefile | 5 - plugins/notifications/http/go.mod | 6 +- plugins/notifications/http/go.sum | 18 +- plugins/notifications/slack/Makefile | 5 - plugins/notifications/slack/go.mod | 6 +- plugins/notifications/slack/go.sum | 18 +- plugins/notifications/splunk/Makefile | 5 - plugins/notifications/splunk/go.mod | 6 +- plugins/notifications/splunk/go.sum | 20 +- rpm/SPECS/crowdsec.spec | 4 - test/README.md | 10 +- test/ansible/debug_tools.yml | 2 + test/ansible/provision_dependencies.yml | 2 + test/ansible/requirements.yml | 4 +- .../ansible/roles/make_fixture/tasks/main.yml | 4 +- test/ansible/vagrant/alma-8/Vagrantfile | 4 + test/ansible/vagrant/alma-9/Vagrantfile | 4 + test/ansible/vagrant/centos-7/Vagrantfile | 2 + test/ansible/vagrant/centos-8/Vagrantfile | 6 +- test/ansible/vagrant/centos-9/Vagrantfile | 4 + test/ansible/vagrant/common | 65 +- .../vagrant/debian-10-buster/Vagrantfile | 2 + .../vagrant/debian-11-bullseye/Vagrantfile | 2 + .../vagrant/debian-12-bookworm/Vagrantfile | 11 + .../vagrant/debian-9-stretch/Vagrantfile | 3 + .../vagrant/debian-testing/Vagrantfile | 2 + .../experimental/hardenedbsd-13/Vagrantfile | 21 +- .../experimental/hardenedbsd-13/bootstrap | 5 - .../experimental/openbsd-6/Vagrantfile | 14 + .../vagrant/experimental/openbsd-6/skip | 9 + .../experimental/openbsd-7/Vagrantfile | 24 +- .../vagrant/experimental/openbsd-7/bootstrap | 6 - test/ansible/vagrant/fedora-33/Vagrantfile | 2 + test/ansible/vagrant/fedora-34/Vagrantfile | 2 + test/ansible/vagrant/fedora-35/Vagrantfile | 2 + test/ansible/vagrant/fedora-36/Vagrantfile | 2 + test/ansible/vagrant/fedora-37/Vagrantfile | 11 + test/ansible/vagrant/fedora-38/Vagrantfile | 10 + test/ansible/vagrant/freebsd-12/Vagrantfile | 3 + test/ansible/vagrant/freebsd-13/Vagrantfile | 3 + test/ansible/vagrant/oracle-7/Vagrantfile | 3 + test/ansible/vagrant/oracle-8/Vagrantfile | 3 + test/ansible/vagrant/oracle-9/Vagrantfile | 3 + test/ansible/vagrant/rocky-8/Vagrantfile | 4 + test/ansible/vagrant/rocky-9/Vagrantfile | 4 + .../vagrant/ubuntu-16.04-xenial/Vagrantfile | 2 + .../vagrant/ubuntu-18.04-bionic/Vagrantfile | 2 + .../vagrant/ubuntu-20.04-focal/Vagrantfile | 3 + .../vagrant/ubuntu-22.04-jammy/Vagrantfile | 2 + .../vagrant/ubuntu-22.10-kinetic/Vagrantfile | 10 + .../vagrant/ubuntu-23.04-lunar/Vagrantfile | 10 + .../vagrant/wizard/centos-8/Vagrantfile | 43 +- .../ansible/vagrant/wizard/centos-8/bootstrap | 5 - test/ansible/vagrant/wizard/common | 67 ++ .../wizard/debian-10-buster/Vagrantfile | 12 + .../wizard/debian-11-bullseye/Vagrantfile | 12 + .../wizard/debian-12-bookworm/Vagrantfile | 12 + .../wizard/debian-bullseye/Vagrantfile | 43 - .../vagrant/wizard/debian-bullseye/bootstrap | 5 - .../vagrant/wizard/debian-buster/Vagrantfile | 43 - .../vagrant/wizard/debian-buster/bootstrap | 5 - .../vagrant/wizard/fedora-36/Vagrantfile | 39 +- .../vagrant/wizard/fedora-36/bootstrap | 5 - .../wizard/ubuntu-22.04-jammy/Vagrantfile | 44 +- .../wizard/ubuntu-22.04-jammy/bootstrap | 5 - .../wizard/ubuntu-22.10-kinetic/Vagrantfile | 44 +- test/ansible/vars/go.yml | 2 +- test/bats-detect/openresty-deb.bats | 7 +- test/bats-detect/openresty-rpm.bats | 8 +- test/bats.mk | 3 +- test/bats/01_crowdsec.bats | 50 +- test/bats/01_cscli.bats | 8 +- test/bats/04_capi.bats | 2 +- test/bats/72_plugin_badconfig.bats | 18 +- test/bats/90_decisions.bats | 121 ++- test/bats/testdata/90_decisions/csv_decisions | 6 + test/bats/testdata/90_decisions/decisions.csv | 6 + .../bats/testdata/90_decisions/decisions.json | 42 + .../bats/testdata/90_decisions/json_decisions | 42 + test/bin/assert-crowdsec-not-running | 11 +- test/bin/check-requirements | 34 +- test/lib/setup_file.sh | 18 + test/tools/.do-not-remove | 1 + 244 files changed, 4676 insertions(+), 2011 deletions(-) create mode 100644 .github/workflows/cache-cleanup.yaml create mode 100644 cmd/crowdsec-cli/copyfile.go create mode 100644 cmd/crowdsec-cli/decisions_import.go create mode 100644 mk/__gmsl create mode 100644 mk/gmsl create mode 100644 mk/gmsl.html create mode 100644 pkg/apiserver/apic_metrics.go create mode 100644 pkg/apiserver/apic_metrics_test.go rename pkg/{types => cwhub}/dataset.go (70%) rename pkg/{types => cwhub}/dataset_test.go (94%) create mode 100644 pkg/leakybucket/bayesian.go create mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml create mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml create mode 100644 pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json create mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml create mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml create mode 100644 pkg/leakybucket/tests/multiple-bayesian-bucket/test.json create mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml create mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml create mode 100644 pkg/leakybucket/tests/simple-bayesian-bucket/test.json rename pkg/{types => parser}/grok_pattern.go (99%) create mode 100644 pkg/types/datasource.go create mode 100644 test/ansible/vagrant/debian-12-bookworm/Vagrantfile delete mode 100755 test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap create mode 100644 test/ansible/vagrant/experimental/openbsd-6/Vagrantfile create mode 100755 test/ansible/vagrant/experimental/openbsd-6/skip delete mode 100755 test/ansible/vagrant/experimental/openbsd-7/bootstrap create mode 100644 test/ansible/vagrant/fedora-37/Vagrantfile create mode 100644 test/ansible/vagrant/fedora-38/Vagrantfile create mode 100644 test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile create mode 100644 test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile delete mode 100755 test/ansible/vagrant/wizard/centos-8/bootstrap create mode 100644 test/ansible/vagrant/wizard/common create mode 100644 test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile create mode 100644 test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile create mode 100644 test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile delete mode 100644 test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile delete mode 100755 test/ansible/vagrant/wizard/debian-bullseye/bootstrap delete mode 100644 test/ansible/vagrant/wizard/debian-buster/Vagrantfile delete mode 100755 test/ansible/vagrant/wizard/debian-buster/bootstrap delete mode 100755 test/ansible/vagrant/wizard/fedora-36/bootstrap delete mode 100755 test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap create mode 100644 test/bats/testdata/90_decisions/csv_decisions create mode 100644 test/bats/testdata/90_decisions/decisions.csv create mode 100644 test/bats/testdata/90_decisions/decisions.json create mode 100644 test/bats/testdata/90_decisions/json_decisions create mode 100644 test/tools/.do-not-remove diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index 026e8feaac2..c62229cbe3c 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -27,40 +27,26 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture + run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 - name: "Run hub tests" run: make bats-test-hub diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml index 3c8825a47d9..529c3f52162 100644 --- a/.github/workflows/bats-mysql.yml +++ b/.github/workflows/bats-mysql.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -34,41 +34,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 env: DB_BACKEND: mysql MYSQL_HOST: 127.0.0.1 diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml index de99a8e22d3..91e7ac10361 100644 --- a/.github/workflows/bats-postgres.yml +++ b/.github/workflows/bats-postgres.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -35,41 +35,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture (DB_BACKEND: pgx)" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 env: DB_BACKEND: pgx PGHOST: 127.0.0.1 diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index 7b2e763b32d..17b2aac0989 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -24,41 +24,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 - name: "Run tests" run: make bats-test diff --git a/.github/workflows/cache-cleanup.yaml b/.github/workflows/cache-cleanup.yaml new file mode 100644 index 00000000000..d193650246b --- /dev/null +++ b/.github/workflows/cache-cleanup.yaml @@ -0,0 +1,35 @@ +# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches + +name: cleanup caches by a branch +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml index b9d9722f3d7..3b66b0c1fe3 100644 --- a/.github/workflows/ci-windows-build-msi.yml +++ b/.github/workflows/ci-windows-build-msi.yml @@ -23,38 +23,27 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: Build runs-on: windows-2019 steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build - run: make windows_installer + run: make windows_installer BUILD_RE2_WASM=1 - name: Upload MSI uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 6476d9f0aa1..913c4766238 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -30,18 +30,6 @@ jobs: with: config: .github/buildkit.toml - # - name: "Build flavor: full" - # uses: docker/build-push-action@v4 - # with: - # context: . - # file: ./Dockerfile - # tags: crowdsecurity/crowdsec:test - # target: full - # platforms: linux/amd64 - # load: true - # cache-from: type=gha - # cache-to: type=gha,mode=min - - name: "Build flavor: slim" uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index b8e81bbe083..500fc58ce99 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -22,39 +22,28 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: windows-2022 steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build run: | - make build + make build BUILD_RE2_WASM=1 - name: Run tests run: | diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 9b4adb4ce69..079f6c827c8 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -34,7 +34,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -110,35 +110,30 @@ jobs: steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - - name: Build and run tests + - name: Build and run tests, static run: | + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev go install github.com/ory/go-acc@v0.2.8 go install github.com/kyoh86/richgo@v0.3.10 set -o pipefail - make build + make build BUILD_STATIC=1 + make go-acc | richgo testfilter + + - name: Run tests again, dynamic + run: | + make clean build make go-acc | richgo testfilter - name: Upload unit coverage to Codecov diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml index 60035573958..54419cc6e51 100644 --- a/.github/workflows/release_publish-package.yml +++ b/.github/workflows/release_publish-package.yml @@ -14,41 +14,32 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: Build and upload binary package runs-on: ubuntu-latest steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build the binaries - run: make release + run: | + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev + make vendor release BUILD_STATIC=1 - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | tag_name="${GITHUB_REF##*/}" - hub release edit -a crowdsec-release.tgz -m "" "$tag_name" + hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name" diff --git a/.gitignore b/.gitignore index 4c5cb0a1ffa..8fe1778baec 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ *.test *.cover +# Test dependencies +test/tools/* + # VMs used for dev/test .vagrant @@ -30,8 +33,10 @@ test/coverage/* *.swp *.swo -# Dependency directories (remove the comment below to include it) -# vendor/ +# Dependencies are not vendored by default, but a tarball is created by "make vendor" +# and provided in the release. Used by freebsd, gentoo, etc. +vendor/ +vendor.tgz # crowdsec binaries cmd/crowdsec-cli/cscli diff --git a/.golangci.yml b/.golangci.yml index 79900ae7df1..faa67c4bb80 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -65,7 +65,6 @@ linters: # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers # - bidichk # Checks for dangerous unicode character sequences # - decorder # check declaration order and count of types, constants, variables and functions - # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases @@ -125,6 +124,7 @@ linters: - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers - wastedassign # wastedassign finds wasted assignment statements. - wrapcheck # Checks that errors returned from external packages are wrapped + - depguard # Go linter that checks if package imports are in a list of acceptable packages # # Recommended? (requires some work) diff --git a/Dockerfile b/Dockerfile index da1c3ab06a9..2073d3c5a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,31 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.4 +ARG GOVERSION=1.20.5 FROM golang:${GOVERSION}-alpine AS build WORKDIR /go/src/crowdsec -COPY . . - -# Alpine does not ship a static version of re2, we can build it ourselves -# Later versions require 'abseil', which is likewise not available in its static form +# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway. ENV RE2_VERSION=2023-03-01 # wizard.sh requires GNU coreutils -RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \ wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ tar -xzf ${RE2_VERSION}.tar.gz && \ cd re2-${RE2_VERSION} && \ - make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - cd - && \ - make clean release DOCKER_BUILD=1 && \ + go install github.com/mikefarah/yq/v4@v4.34.1 + +COPY . . + +RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists && \ - go install github.com/mikefarah/yq/v4@v4.31.2 + cscli parsers install crowdsecurity/whitelists # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete diff --git a/Dockerfile.debian b/Dockerfile.debian index 10b06befdf0..61778cd9eb4 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,32 +1,41 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.4 +ARG GOVERSION=1.20.5 -FROM golang:${GOVERSION}-bullseye AS build +FROM golang:${GOVERSION}-bookworm AS build WORKDIR /go/src/crowdsec -COPY . . - ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" +# We like to choose the release of re2 to use, the debian version is usually older. +ENV RE2_VERSION=2023-03-01 + # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ + wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ + tar -xzf ${RE2_VERSION}.tar.gz && \ + cd re2-${RE2_VERSION} && \ + make && \ + make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - make clean release DOCKER_BUILD=1 && \ + go install github.com/mikefarah/yq/v4@v4.34.1 + +COPY . . + +RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists && \ - go install github.com/mikefarah/yq/v4@v4.31.2 + cscli parsers install crowdsecurity/whitelists # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete -FROM debian:bullseye-slim as slim +FROM debian:bookworm-slim as slim ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" @@ -44,9 +53,6 @@ RUN apt-get update && \ mkdir -p /staging/var/lib/crowdsec && \ mkdir -p /var/lib/crowdsec/data -RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \ - && apt-get update && apt-get install -t bullseye-backports -y libsystemd0 - COPY --from=build /go/bin/yq /usr/local/bin/yq COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec diff --git a/Makefile b/Makefile index d6f1b95f237..0fb36261ffe 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,36 @@ include mk/platform.mk +include mk/gmsl +# By default, this build requires the C++ re2 library to be installed. +# +# Debian/Ubuntu: apt install libre2-dev +# Fedora/CentOS: dnf install re2-devel +# FreeBSD: pkg install re2 +# Alpine: apk add re2-dev +# Windows: choco install re2 +# MacOS: brew install re2 + +# To build without re2, run "make BUILD_RE2_WASM=1" +# The WASM version is slower and introduces a short delay when starting a process +# (including cscli) so it is not recommended for production use. +BUILD_RE2_WASM ?= 0 + +# To build static binaries, run "make BUILD_STATIC=1". +# On some platforms, this requires additional packages +# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository). +# If the static build fails at the link stage, it might be because the static library is not provided +# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it. +BUILD_STATIC ?= 0 + +# List of plugins to build +PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) + +# Can be overriden, if you can deal with the consequences BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 +#-------------------------------------- + GOCMD = go GOTEST = $(GOCMD) test @@ -10,8 +38,6 @@ BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec CSCLI_FOLDER = ./cmd/crowdsec-cli/ - -PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) @@ -22,8 +48,14 @@ RELDIR = crowdsec-$(BUILD_VERSION) GO_MODULE_NAME = github.com/crowdsecurity/crowdsec -# see if we have libre2-dev installed for C++ optimizations -RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) +# Check if a given value is considered truthy and returns "0" or "1". +# A truthy value is one of the following: "1", "yes", or "true", case-insensitive. +# +# Usage: +# ifeq ($(call bool,$(FOO)),1) +# $(info Let's foo) +# endif +bool = $(if $(filter $(call lc, $1),1 yes true),1,0) #-------------------------------------- # @@ -46,13 +78,30 @@ endif GO_TAGS := netgo,osusergo,sqlite_omit_load_extension -ifneq (,$(RE2_CHECK)) +ifeq ($(call bool,$(BUILD_RE2_WASM)),0) +ifeq ($(PKG_CONFIG),) + $(error "pkg-config is not available. Please install pkg-config.") +endif + +ifeq ($(RE2_CHECK),) +# we could detect the platform and suggest the command to install +RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version" +else # += adds a space that we don't want GO_TAGS := $(GO_TAGS),re2_cgo LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif +endif + +ifeq ($(call bool,$(BUILD_STATIC)),1) +BUILD_TYPE = static +EXTLDFLAGS := -extldflags '-static' +else +BUILD_TYPE = dynamic +EXTLDFLAGS := +endif -export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ +export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) @@ -64,12 +113,15 @@ endif .PHONY: build build: pre-build goversion crowdsec cscli plugins +# Sanity checks and build information .PHONY: pre-build pre-build: -ifdef BUILD_STATIC - $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) + $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH)) + +ifneq (,$(RE2_FAIL)) + $(error $(RE2_FAIL)) endif - $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) + ifneq (,$(RE2_CHECK)) $(info Using C++ regexp library) else @@ -113,6 +165,7 @@ testclean: bats-clean @$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR) @$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR) +# for the tests with localstack export AWS_ENDPOINT_FORCE=http://localhost:4566 export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -120,15 +173,18 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY testenv: @echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)' +# run the tests with localstack .PHONY: test test: testenv goversion $(GOTEST) $(LD_OPTS) ./... +# run the tests with localstack and coverage .PHONY: go-acc go-acc: testenv goversion go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \ sed 's/ *coverage:.*of statements in.*//' +# mock AWS services .PHONY: localstack localstack: docker-compose -f test/localstack/docker-compose.yml up @@ -137,13 +193,27 @@ localstack: localstack-stop: docker-compose -f test/localstack/docker-compose.yml down +# list of plugins that contain go.mod +PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi)) + +# build vendor.tgz to be distributed with the release .PHONY: vendor vendor: - @echo "Vendoring dependencies" - @$(GOCMD) mod vendor - @$(foreach plugin,$(PLUGINS), \ - $(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \ + $(foreach plugin_dir,$(PLUGIN_VENDOR), \ + cd $(plugin_dir) >/dev/null && \ + $(GOCMD) mod vendor && \ + cd - >/dev/null; \ + ) + $(GOCMD) mod vendor + tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor) + +# remove vendor directories and vendor.tgz +.PHONY: vendor-remove +vendor-remove: + $(foreach plugin_dir,$(PLUGIN_VENDOR), \ + $(RM) $(plugin_dir)/vendor; \ ) + $(RM) vendor vendor.tgz .PHONY: package package: @@ -174,13 +244,16 @@ else @if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; } endif +# build a release tarball .PHONY: release release: check_release build package +# build the windows installer .PHONY: windows_installer windows_installer: build @.\make_installer.ps1 -version $(BUILD_VERSION) +# build the chocolatey package .PHONY: chocolatey chocolatey: windows_installer @.\make_chocolatey.ps1 -version $(BUILD_VERSION) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c529ee2faa7..b1564b375f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ stages: - task: GoTool@0 displayName: "Install Go 1.20" inputs: - version: '1.20.4' + version: '1.20.5' - pwsh: | choco install -y make @@ -38,7 +38,7 @@ stages: pwsh: true #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI script: | - make build + make build BUILD_RE2_WASM=1 - task: AzureKeyVault@2 inputs: azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)' diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 25cb26515d7..6abe3db5afc 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -15,7 +15,6 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -155,7 +154,6 @@ var alertTemplate = ` ` - func DisplayOneAlert(alert *models.Alert, withDetail bool) error { if csConfig.Cscli.Output == "human" { tmpl, err := template.New("alert").Parse(alertTemplate) @@ -211,11 +209,11 @@ func NewAlertsCmd() *cobra.Command { PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "parsing api url %s", apiURL) + return fmt.Errorf("parsing api url %s: %w", apiURL, err) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -226,7 +224,7 @@ func NewAlertsCmd() *cobra.Command { }) if err != nil { - return errors.Wrap(err, "new api client") + return fmt.Errorf("new api client: %w", err) } return nil }, diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index e67d33ce458..af6e9c2e86f 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -6,6 +6,11 @@ import ( "net/url" "os" + "github.com/go-openapi/strfmt" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -14,11 +19,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" ) const CAPIBaseURL string = "https://api.crowdsec.net/" @@ -31,8 +31,11 @@ func NewCapiCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { - return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine") + if err := csConfig.LoadAPIServer(); err != nil { + return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err) + } + if csConfig.DisableAPI { + return nil } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath) @@ -133,7 +136,7 @@ func NewCapiStatusCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { var err error if csConfig.API.Server == nil { - log.Fatalln("There is no configuration on 'api.server:'") + log.Fatal("There is no configuration on 'api.server:'") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 30cf729fe1c..717fc990b9b 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -5,11 +5,9 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -34,17 +32,17 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) if _, err := os.Stat(parentDir); err != nil { - return errors.Wrapf(err, "while checking parent directory %s existence", parentDir) + return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) } if err = os.Mkdir(dirPath, 0o700); err != nil { - return errors.Wrapf(err, "while creating %s", dirPath) + return fmt.Errorf("while creating %s: %w", dirPath, err) } if csConfig.ConfigPaths.SimulationFilePath != "" { backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation) + if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { + return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err) } log.Infof("Saved simulation to %s", backupSimulation) @@ -56,14 +54,14 @@ func backupConfigToDirectory(dirPath string) error { */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) + if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } } acquisBackupDir := filepath.Join(dirPath, "acquis") if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err) + return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err) } if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { @@ -75,11 +73,11 @@ func backupConfigToDirectory(dirPath string) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) + if err = CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) } log.Infof("Saved acquis %s to %s", acquisFile, targetFname) @@ -88,8 +86,8 @@ func backupConfigToDirectory(dirPath string) error { if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err) + if err = CopyFile(ConfigFilePath, backupMain); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err) } log.Infof("Saved default yaml to %s", backupMain) @@ -97,8 +95,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) + if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) } log.Infof("Saved online API credentials to %s", backupCAPICreds) @@ -106,8 +104,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) + if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) } log.Infof("Saved local API credentials to %s", backupLAPICreds) @@ -115,21 +113,20 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" { backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) + if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) } log.Infof("Saved profiles to %s", backupProfiles) } if err = BackupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config : %s", err) + return fmt.Errorf("failed to backup hub config: %s", err) } return nil } - func runConfigBackup(cmd *cobra.Command, args []string) error { if err := csConfig.LoadHub(); err != nil { return err @@ -147,7 +144,6 @@ func runConfigBackup(cmd *cobra.Command, args []string) error { return nil } - func NewConfigBackupCmd() *cobra.Command { cmdConfigBackup := &cobra.Command{ Use: `backup "directory"`, diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 79d36d428f3..55ab7aa9bad 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -7,13 +7,11 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -38,7 +36,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if _, err = os.Stat(backupMain); err == nil { if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" { - if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { + if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err) } } @@ -51,21 +49,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) if _, err = os.Stat(backupCAPICreds); err == nil { - if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { + if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err) } } backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) if _, err = os.Stat(backupLAPICreds); err == nil { - if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { + if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err) } } backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) if _, err = os.Stat(backupProfiles); err == nil { - if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { + if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err) } } @@ -106,7 +104,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) if _, err = os.Stat(backupSimulation); err == nil { - if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { + if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err) } } @@ -123,7 +121,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if _, err = os.Stat(backupAcquisition); err == nil { log.Debugf("restoring backup'ed %s", backupAcquisition) - if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { + if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } @@ -134,12 +132,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { for _, acquisFile := range acquisFiles { targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) } log.Debugf("restoring %s to %s", acquisFile, targetFname) - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } } @@ -157,10 +155,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } @@ -175,7 +173,6 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { return nil } - func runConfigRestore(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -200,7 +197,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return nil } - func NewConfigRestoreCmd() *cobra.Command { cmdConfigRestore := &cobra.Command{ Use: `restore "directory"`, diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go new file mode 100644 index 00000000000..4de6cd6e24a --- /dev/null +++ b/cmd/crowdsec-cli/copyfile.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + + +/*help to copy the file, ioutil doesn't offer the feature*/ + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +/*copy the file, ioutile doesn't offer the feature*/ +func CopyFile(sourceSymLink, destinationFile string) (err error) { + sourceFile, err := filepath.EvalSymlinks(sourceSymLink) + if err != nil { + log.Infof("Not a symlink : %s", err) + sourceFile = sourceSymLink + } + + sourceFileStat, err := os.Stat(sourceFile) + if err != nil { + return + } + if !sourceFileStat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) + } + destinationFileStat, err := os.Stat(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(destinationFileStat.Mode().IsRegular()) { + return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) + } + if os.SameFile(sourceFileStat, destinationFileStat) { + return + } + } + if err = os.Link(sourceFile, destinationFile); err != nil { + err = copyFileContents(sourceFile, destinationFile) + } + return +} + diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index f2f3efcf822..ce3d0e46e2b 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -7,19 +7,15 @@ import ( "fmt" "net/url" "os" - "path/filepath" "strconv" "strings" "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/jszwec/csvutil" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -112,12 +108,12 @@ func NewDecisionsCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } password := strfmt.Password(csConfig.API.Client.Credentials.Password) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL) + return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -127,7 +123,7 @@ func NewDecisionsCmd() *cobra.Command { VersionPrefix: "v1", }) if err != nil { - return errors.Wrap(err, "creating api client") + return fmt.Errorf("creating api client: %w", err) } return nil }, @@ -169,11 +165,11 @@ cscli decisions list -t ban `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error /*take care of shorthand options*/ - if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { - log.Fatalf("%s", err) + if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { + return err } filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true @@ -189,7 +185,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Until = fmt.Sprintf("%d%s", days*24, "h") } @@ -202,7 +198,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) } *filter.Since = fmt.Sprintf("%d%s", days*24, "h") } @@ -238,13 +234,15 @@ cscli decisions list -t ban alerts, _, err := Client.Alerts.List(context.Background(), filter) if err != nil { - log.Fatalf("Unable to list decisions : %v", err) + return fmt.Errorf("unable to retrieve decisions: %w", err) } err = DecisionsToTable(alerts, printMachine) if err != nil { - log.Fatalf("unable to list decisions : %v", err) + return fmt.Errorf("unable to print decisions: %w", err) } + + return nil }, } cmdDecisionsList.Flags().SortFlags = false @@ -288,7 +286,7 @@ cscli decisions add --scope username --value foobar /*TBD : fix long and example*/ Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error alerts := models.AddAlertsRequest{} origin := types.CscliOrigin @@ -303,7 +301,7 @@ cscli decisions add --scope username --value foobar /*take care of shorthand options*/ if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil { - log.Fatalf("%s", err) + return err } if addIP != "" { @@ -314,7 +312,7 @@ cscli decisions add --scope username --value foobar addScope = types.Range } else if addValue == "" { printHelp(cmd) - log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)") + return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)") } if addReason == "" { @@ -357,10 +355,11 @@ cscli decisions add --scope username --value foobar _, _, err = Client.Alerts.Add(context.Background(), alerts) if err != nil { - log.Fatal(err) + return err } log.Info("Decision successfully added") + return nil }, } @@ -401,25 +400,27 @@ cscli decisions delete --id 42 cscli decisions delete --type captcha `, /*TBD : refaire le Long/Example*/ - PreRun: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { if delDecisionAll { - return + return nil } if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && *delFilter.OriginEquals == "" && delDecisionId == "" { cmd.Usage() - log.Fatalln("At least one filter or --all must be specified") + return fmt.Errorf("at least one filter or --all must be specified") } + + return nil }, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error var decisions *models.DeleteDecisionResponse /*take care of shorthand options*/ - if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { - log.Fatalf("%s", err) + if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { + return err } if *delFilter.ScopeEquals == "" { delFilter.ScopeEquals = nil @@ -449,18 +450,19 @@ cscli decisions delete --type captcha if delDecisionId == "" { decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter) if err != nil { - log.Fatalf("Unable to delete decisions : %v", err) + return fmt.Errorf("Unable to delete decisions: %v", err) } } else { if _, err = strconv.Atoi(delDecisionId); err != nil { - log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err) + return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err) } decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId) if err != nil { - log.Fatalf("Unable to delete decision : %v", err) + return fmt.Errorf("Unable to delete decision: %v", err) } } log.Infof("%s decision(s) deleted", decisions.NbDeleted) + return nil }, } @@ -478,192 +480,3 @@ cscli decisions delete --type captcha return cmdDecisionsDelete } - -func NewDecisionsImportCmd() *cobra.Command { - var ( - defaultDuration = "4h" - defaultScope = "ip" - defaultType = "ban" - defaultReason = "manual" - importDuration string - importScope string - importReason string - importType string - importFile string - batchSize int - ) - - var cmdDecisionImport = &cobra.Command{ - Use: "import [options]", - Short: "Import decisions from json or csv file", - Long: "expected format :\n" + - "csv : any of duration,origin,reason,scope,type,value, with a header line\n" + - `json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, - DisableAutoGenTag: true, - Example: `decisions.csv : -duration,scope,value -24h,ip,1.2.3.4 - -cscsli decisions import -i decisions.csv - -decisions.json : -[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] -`, - Run: func(cmd *cobra.Command, args []string) { - if importFile == "" { - log.Fatalf("Please provide a input file containing decisions with -i flag") - } - csvData, err := os.ReadFile(importFile) - if err != nil { - log.Fatalf("unable to open '%s': %s", importFile, err) - } - type decisionRaw struct { - Duration string `csv:"duration,omitempty" json:"duration,omitempty"` - Origin string `csv:"origin,omitempty" json:"origin,omitempty"` - Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` - Scope string `csv:"scope,omitempty" json:"scope,omitempty"` - Type string `csv:"type,omitempty" json:"type,omitempty"` - Value string `csv:"value" json:"value"` - } - var decisionsListRaw []decisionRaw - switch fileFormat := filepath.Ext(importFile); fileFormat { - case ".json": - if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil { - log.Fatalf("unable to unmarshall json: '%s'", err) - } - case ".csv": - if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil { - log.Fatalf("unable to unmarshall csv: '%s'", err) - } - default: - log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile) - } - - decisionsList := make([]*models.Decision, 0) - for i, decisionLine := range decisionsListRaw { - line := i + 2 - if decisionLine.Value == "" { - log.Fatalf("please provide a 'value' in your csv line %d", line) - } - /*deal with defaults and cli-override*/ - if decisionLine.Duration == "" { - decisionLine.Duration = defaultDuration - log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration) - } - if importDuration != "" { - decisionLine.Duration = importDuration - log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration) - } - decisionLine.Origin = types.CscliImportOrigin - - if decisionLine.Scenario == "" { - decisionLine.Scenario = defaultReason - log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario) - } - if importReason != "" { - decisionLine.Scenario = importReason - log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason) - } - if decisionLine.Type == "" { - decisionLine.Type = defaultType - log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type) - } - if importType != "" { - decisionLine.Type = importType - log.Debugf("'type' line %d, using supplied value: '%s'", line, importType) - } - if decisionLine.Scope == "" { - decisionLine.Scope = defaultScope - log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope) - } - if importScope != "" { - decisionLine.Scope = importScope - log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope) - } - decision := models.Decision{ - Value: ptr.Of(decisionLine.Value), - Duration: ptr.Of(decisionLine.Duration), - Origin: ptr.Of(decisionLine.Origin), - Scenario: ptr.Of(decisionLine.Scenario), - Type: ptr.Of(decisionLine.Type), - Scope: ptr.Of(decisionLine.Scope), - Simulated: new(bool), - } - decisionsList = append(decisionsList, &decision) - } - alerts := models.AddAlertsRequest{} - - if batchSize > 0 { - for i := 0; i < len(decisionsList); i += batchSize { - end := i + batchSize - if end > len(decisionsList) { - end = len(decisionsList) - } - decisionBatch := decisionsList[i:end] - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))), - - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(decisionBatch))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: decisionBatch, - } - alerts = append(alerts, &importAlert) - } - } else { - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))), - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(decisionsList))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: decisionsList, - } - alerts = append(alerts, &importAlert) - } - - if len(decisionsList) > 1000 { - log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList)) - } - - _, _, err = Client.Alerts.Add(context.Background(), alerts) - if err != nil { - log.Fatal(err) - } - log.Infof("%d decisions successfully imported", len(decisionsList)) - }, - } - - cmdDecisionImport.Flags().SortFlags = false - cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file") - cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)") - cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)") - cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)") - cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)") - cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions") - - return cmdDecisionImport -} diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go new file mode 100644 index 00000000000..6a47a96b3ea --- /dev/null +++ b/cmd/crowdsec-cli/decisions_import.go @@ -0,0 +1,272 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/jszwec/csvutil" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" + "github.com/crowdsecurity/go-cs-lib/pkg/slicetools" + + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +// decisionRaw is only used to unmarshall json/csv decisions +type decisionRaw struct { + Duration string `csv:"duration,omitempty" json:"duration,omitempty"` + Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` + Scope string `csv:"scope,omitempty" json:"scope,omitempty"` + Type string `csv:"type,omitempty" json:"type,omitempty"` + Value string `csv:"value" json:"value"` +} + +func parseDecisionList(content []byte, format string) ([]decisionRaw, error) { + ret := []decisionRaw{} + + switch format { + case "values": + log.Infof("Parsing values") + scanner := bufio.NewScanner(bytes.NewReader(content)) + for scanner.Scan() { + value := strings.TrimSpace(scanner.Text()) + ret = append(ret, decisionRaw{Value: value}) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("unable to parse values: '%s'", err) + } + case "json": + log.Infof("Parsing json") + if err := json.Unmarshal(content, &ret); err != nil { + return nil, err + } + case "csv": + log.Infof("Parsing csv") + if err := csvutil.Unmarshal(content, &ret); err != nil { + return nil, fmt.Errorf("unable to parse csv: '%s'", err) + } + default: + return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format) + } + + return ret, nil +} + + +func runDecisionsImport(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + input, err := flags.GetString("input") + if err != nil { + return err + } + + defaultDuration, err := flags.GetString("duration") + if err != nil { + return err + } + if defaultDuration == "" { + return fmt.Errorf("--duration cannot be empty") + } + + defaultScope, err := flags.GetString("scope") + if err != nil { + return err + } + if defaultScope == "" { + return fmt.Errorf("--scope cannot be empty") + } + + defaultReason, err := flags.GetString("reason") + if err != nil { + return err + } + if defaultReason == "" { + return fmt.Errorf("--reason cannot be empty") + } + + defaultType, err := flags.GetString("type") + if err != nil { + return err + } + if defaultType == "" { + return fmt.Errorf("--type cannot be empty") + } + + batchSize, err := flags.GetInt("batch") + if err != nil { + return err + } + + format, err := flags.GetString("format") + if err != nil { + return err + } + + var ( + content []byte + fin *os.File + ) + + // set format if the file has a json or csv extension + if format == "" { + if strings.HasSuffix(input, ".json") { + format = "json" + } else if strings.HasSuffix(input, ".csv") { + format = "csv" + } + } + + if format == "" { + return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag") + } + + if input == "-" { + fin = os.Stdin + input = "stdin" + } else { + fin, err = os.Open(input) + if err != nil { + return fmt.Errorf("unable to open %s: %s", input, err) + } + } + + content, err = io.ReadAll(fin) + if err != nil { + return fmt.Errorf("unable to read from %s: %s", input, err) + } + + decisionsListRaw, err := parseDecisionList(content, format) + if err != nil { + return err + } + + decisions := make([]*models.Decision, len(decisionsListRaw)) + for i, d := range decisionsListRaw { + if d.Value == "" { + return fmt.Errorf("item %d: missing 'value'", i) + } + + if d.Duration == "" { + d.Duration = defaultDuration + log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration) + } + + if d.Scenario == "" { + d.Scenario = defaultReason + log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason) + } + + if d.Type == "" { + d.Type = defaultType + log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType) + } + + if d.Scope == "" { + d.Scope = defaultScope + log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope) + } + + decisions[i] = &models.Decision{ + Value: ptr.Of(d.Value), + Duration: ptr.Of(d.Duration), + Origin: ptr.Of(types.CscliImportOrigin), + Scenario: ptr.Of(d.Scenario), + Type: ptr.Of(d.Type), + Scope: ptr.Of(d.Scope), + Simulated: ptr.Of(false), + } + } + + alerts := models.AddAlertsRequest{} + + for _, chunk := range slicetools.Chunks(decisions, batchSize) { + log.Debugf("Processing chunk of %d decisions", len(chunk)) + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))), + + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(chunk))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: chunk, + } + alerts = append(alerts, &importAlert) + } + + if len(decisions) > 1000 { + log.Infof("You are about to add %d decisions, this may take a while", len(decisions)) + } + + _, _, err = Client.Alerts.Add(context.Background(), alerts) + if err != nil { + return err + } + + log.Infof("Imported %d decisions", len(decisions)) + return nil +} + + +func NewDecisionsImportCmd() *cobra.Command { + var cmdDecisionsImport = &cobra.Command{ + Use: "import [options]", + Short: "Import decisions from a file or pipe", + Long: "expected format:\n" + + "csv : any of duration,reason,scope,type,value, with a header line\n" + + `json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, + DisableAutoGenTag: true, + Example: `decisions.csv: +duration,scope,value +24h,ip,1.2.3.4 + +$ cscli decisions import -i decisions.csv + +decisions.json: +[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] + +The file format is detected from the extension, but can be forced with the --format option +which is required when reading from standard input. + +Raw values, standard input: + +$ echo "1.2.3.4" | cscli decisions import -i - --format values +`, + RunE: runDecisionsImport, + } + + flags := cmdDecisionsImport.Flags() + flags.SortFlags = false + flags.StringP("input", "i", "", "Input file") + flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m") + flags.String("scope", types.Ip, "Decision scope: ip,range,username") + flags.StringP("reason", "R", "manual", "Decision reason: ") + flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle") + flags.Int("batch", 0, "Split import in batches of N decisions") + flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)") + + cmdDecisionsImport.MarkFlagRequired("input") + + return cmdDecisionsImport +} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index e8a29d9cdbc..d03662c4896 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -24,7 +23,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" - "github.com/crowdsecurity/crowdsec/pkg/types" ) var LAPIURLPrefix string = "v1" @@ -205,7 +203,7 @@ func NewLapiCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } return nil }, @@ -440,7 +438,7 @@ cscli lapi context delete --value evt.Line.Src return cmdContext } -func detectStaticField(GrokStatics []types.ExtraField) []string { +func detectStaticField(GrokStatics []parser.ExtraField) []string { ret := make([]string, 0) for _, static := range GrokStatics { if static.Parsed != "" { diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 25bd5acec40..21594310230 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -12,7 +12,6 @@ import ( "time" "github.com/AlecAivazis/survey/v2" - "github.com/enescakir/emoji" "github.com/fatih/color" "github.com/go-openapi/strfmt" "github.com/google/uuid" @@ -85,22 +84,21 @@ func generateID(prefix string) (string, error) { return prefix + suffix, nil } -func displayLastHeartBeat(m *ent.Machine, fancy bool) string { - var hbDisplay string +// getLastHeartbeat returns the last heartbeat timestamp of a machine +// and a boolean indicating if the machine is considered active or not. +func getLastHeartbeat(m *ent.Machine) (string, bool) { + if m.LastHeartbeat == nil { + return "-", false + } - if m.LastHeartbeat != nil { - lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat) - hbDisplay = lastHeartBeat.Truncate(time.Second).String() - if fancy && lastHeartBeat > 2*time.Minute { - hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String()) - } - } else { - hbDisplay = "-" - if fancy { - hbDisplay = emoji.Warning.String() + " -" - } + elapsed := time.Now().UTC().Sub(*m.LastHeartbeat) + + hb := elapsed.Truncate(time.Second).String() + if elapsed > 2*time.Minute { + return hb, false } - return hbDisplay + + return hb, true } func getAgents(out io.Writer, dbClient *database.Client) error { @@ -130,9 +128,10 @@ func getAgents(out io.Writer, dbClient *database.Client) error { } else { validated = "false" } - err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)}) + hb, _ := getLastHeartbeat(m) + err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}) if err != nil { - return fmt.Errorf("failed to write raw output : %s", err) + return fmt.Errorf("failed to write raw output: %w", err) } } csvwriter.Flush() diff --git a/cmd/crowdsec-cli/machines_table.go b/cmd/crowdsec-cli/machines_table.go index cc15bb51bd3..e166fb785a6 100644 --- a/cmd/crowdsec-cli/machines_table.go +++ b/cmd/crowdsec-cli/machines_table.go @@ -24,7 +24,11 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) { validated = emoji.Prohibited.String() } - t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true)) + hb, active := getLastHeartbeat(m) + if !active { + hb = emoji.Warning.String() + " " + hb + } + t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb) } t.Render() diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index ebbefd6b722..fe4c14f27c5 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -15,7 +15,6 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/tomb.v2" @@ -28,14 +27,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csprofiles" ) - type NotificationsCfg struct { Config csplugin.PluginConfig `json:"plugin_config"` Profiles []*csconfig.ProfileCfg `json:"associated_profiles"` ids []uint } - func NewNotificationsCmd() *cobra.Command { var cmdNotifications = &cobra.Command{ Use: "notifications [action]", @@ -57,7 +54,6 @@ func NewNotificationsCmd() *cobra.Command { }, } - cmdNotifications.AddCommand(NewNotificationsListCmd()) cmdNotifications.AddCommand(NewNotificationsInspectCmd()) cmdNotifications.AddCommand(NewNotificationsReinjectCmd()) @@ -65,18 +61,17 @@ func NewNotificationsCmd() *cobra.Command { return cmdNotifications } - func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { pcfgs := map[string]csplugin.PluginConfig{} wf := func(path string, info fs.FileInfo, err error) error { if info == nil { - return errors.Wrapf(err, "error while traversing directory %s", path) + return fmt.Errorf("error while traversing directory %s: %w", path, err) } name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) { ts, err := csplugin.ParsePluginConfigFile(name) if err != nil { - return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name) + return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err) } for _, t := range ts { pcfgs[t.Name] = t @@ -86,14 +81,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { } if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil { - return nil, errors.Wrap(err, "Loading notifification plugin configuration") + return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err) } // A bit of a tricky stuf now: reconcile profiles and notification plugins ncfgs := map[string]NotificationsCfg{} profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return nil, errors.Wrap(err, "Cannot extract profiles from configuration") + return nil, fmt.Errorf("while extracting profiles from configuration: %w", err) } for profileID, profile := range profiles { loop: @@ -129,7 +124,6 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { return ncfgs, nil } - func NewNotificationsListCmd() *cobra.Command { var cmdNotificationsList = &cobra.Command{ Use: "list", @@ -141,7 +135,7 @@ func NewNotificationsListCmd() *cobra.Command { RunE: func(cmd *cobra.Command, arg []string) error { ncfgs, err := getNotificationsConfiguration() if err != nil { - return errors.Wrap(err, "Can't build profiles configuration") + return fmt.Errorf("can't build profiles configuration: %w", err) } if csConfig.Cscli.Output == "human" { @@ -149,14 +143,14 @@ func NewNotificationsListCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(ncfgs, "", " ") if err != nil { - return errors.New("failed to marshal notification configuration") + return fmt.Errorf("failed to marshal notification configuration: %w", err) } fmt.Printf("%s", string(x)) } else if csConfig.Cscli.Output == "raw" { csvwriter := csv.NewWriter(os.Stdout) err := csvwriter.Write([]string{"Name", "Type", "Profile name"}) if err != nil { - return errors.Wrap(err, "failed to write raw header") + return fmt.Errorf("failed to write raw header: %w", err) } for _, b := range ncfgs { profilesList := []string{} @@ -165,7 +159,7 @@ func NewNotificationsListCmd() *cobra.Command { } err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")}) if err != nil { - return errors.Wrap(err, "failed to write raw content") + return fmt.Errorf("failed to write raw content: %w", err) } } csvwriter.Flush() @@ -177,7 +171,6 @@ func NewNotificationsListCmd() *cobra.Command { return cmdNotificationsList } - func NewNotificationsInspectCmd() *cobra.Command { var cmdNotificationsInspect = &cobra.Command{ Use: "inspect", @@ -195,14 +188,14 @@ func NewNotificationsInspectCmd() *cobra.Command { pluginName := arg[0] if pluginName == "" { - errors.New("Please provide a plugin name to inspect") + return fmt.Errorf("please provide a plugin name to inspect") } ncfgs, err := getNotificationsConfiguration() if err != nil { - return errors.Wrap(err, "Can't build profiles configuration") + return fmt.Errorf("can't build profiles configuration: %w", err) } if cfg, ok = ncfgs[pluginName]; !ok { - return errors.New("The provided plugin name doesn't exist or isn't active") + return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName) } if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" { @@ -216,7 +209,7 @@ func NewNotificationsInspectCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(cfg, "", " ") if err != nil { - return errors.New("failed to marshal notification configuration") + return fmt.Errorf("failed to marshal notification configuration: %w", err) } fmt.Printf("%s", string(x)) } @@ -227,7 +220,6 @@ func NewNotificationsInspectCmd() *cobra.Command { return cmdNotificationsInspect } - func NewNotificationsReinjectCmd() *cobra.Command { var remediation bool var alertOverride string @@ -250,26 +242,26 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not ) if len(args) != 1 { printHelp(cmd) - return errors.New("Wrong number of argument: there should be one argument") + return fmt.Errorf("wrong number of argument: there should be one argument") } //first: get the alert id, err := strconv.Atoi(args[0]) if err != nil { - return errors.New(fmt.Sprintf("bad alert id %s", args[0])) + return fmt.Errorf("bad alert id %s", args[0]) } if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrapf(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } if csConfig.API.Client == nil { - return errors.New("There is no configuration on 'api_client:'") + return fmt.Errorf("missing configuration on 'api_client:'") } if csConfig.API.Client.Credentials == nil { - return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)) + return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "error parsing the URL of the API") + return fmt.Errorf("error parsing the URL of the API: %w", err) } client, err := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -279,16 +271,16 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not VersionPrefix: "v1", }) if err != nil { - return errors.Wrapf(err, "error creating the client for the API") + return fmt.Errorf("error creating the client for the API: %w", err) } alert, _, err := client.Alerts.GetByID(context.Background(), id) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0])) + return fmt.Errorf("can't find alert with id %s: %w", args[0], err) } if alertOverride != "" { if err = json.Unmarshal([]byte(alertOverride), alert); err != nil { - return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag") + return fmt.Errorf("can't unmarshal data in the alert flag: %w", err) } } if !remediation { @@ -298,7 +290,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not // second we start plugins err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths) if err != nil { - return errors.Wrapf(err, "Can't initialize plugins") + return fmt.Errorf("can't initialize plugins: %w", err) } pluginTomb.Go(func() error { @@ -310,13 +302,13 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return errors.Wrap(err, "Cannot extract profiles from configuration") + return fmt.Errorf("cannot extract profiles from configuration: %w", err) } for id, profile := range profiles { _, matched, err := profile.EvaluateProfile(alert) if err != nil { - return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name) + return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err) } if !matched { log.Infof("The profile %s didn't match", profile.Cfg.Name) @@ -344,7 +336,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not } // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(errors.New("terminating")) + pluginTomb.Kill(fmt.Errorf("terminating")) pluginTomb.Wait() return nil }, diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index a5b433228b6..de52dcb4876 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/fatih/color" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -33,7 +32,7 @@ cscli scenarios remove crowdsecurity/ssh-bf } if err := cwhub.SetHubBranch(); err != nil { - return errors.Wrap(err, "while setting hub branch") + return fmt.Errorf("while setting hub branch: %w", err) } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 013abf4b260..66c1493a4b6 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -26,7 +26,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -48,6 +47,14 @@ const ( SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" ) +// from https://github.com/acarl005/stripansi +var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") + +func stripAnsiString(str string) string { + // the byte version doesn't strip correctly + return reStripAnsi.ReplaceAllString(str, "") +} + func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") err := csConfig.LoadPrometheus() @@ -400,7 +407,7 @@ cscli support dump -f /tmp/crowdsec-support.zip log.Errorf("Could not add zip entry for %s: %s", filename, err) continue } - fw.Write([]byte(types.StripAnsiString(string(data)))) + fw.Write([]byte(stripAnsiString(string(data)))) } err = zipWriter.Close() diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 1a99c2acfa5..f77342cc941 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", tfile.Name()) sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", file.Name()) sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -657,7 +657,7 @@ func BackupHub(dirPath string) error { } clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = types.CopyFile(v.LocalPath, tfile); err != nil { + if err = CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) } clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index 3ce249d4cb2..fd2e2ce088c 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "runtime" "time" @@ -20,7 +21,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { apiServer, err := apiserver.NewServer(cConfig.API.Server) if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run local API: %w", err) } if hasPlugins(cConfig.API.Server.Profiles) { @@ -29,23 +30,27 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { if cConfig.PluginConfig == nil && runtime.GOOS != "windows" { return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration") } + if cConfig.ConfigPaths.NotificationDir == "" { return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined") } + if cConfig.ConfigPaths.PluginDir == "" { return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined") } + err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths) if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run plugin broker: %w", err) } + log.Info("initiated plugin broker") apiServer.AttachPluginBroker(&pluginBroker) } err = apiServer.InitController() if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run local API: %w", err) } return apiServer, nil diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 8b4487e158b..68a7c6180da 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -3,10 +3,12 @@ package main import ( "fmt" "os" + "path/filepath" "sync" "time" - "path/filepath" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/go-cs-lib/pkg/trace" @@ -16,31 +18,28 @@ import ( leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error // Populate cwhub package tools - if err := cwhub.GetHubIdx(cConfig.Hub); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err) + if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { + return nil, fmt.Errorf("while loading hub index: %w", err) } // Start loading configs csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) + return nil, fmt.Errorf("while loading parsers: %w", err) } if err := LoadBuckets(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err) + return nil, fmt.Errorf("while loading scenarios: %w", err) } if err := LoadAcquisition(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err) + return nil, fmt.Errorf("while loading acquisition config: %w", err) } return csParsers, nil } @@ -118,7 +117,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { aggregated = true } if err := acquisition.GetMetrics(dataSources, aggregated); err != nil { - return errors.Wrap(err, "while fetching prometheus metrics for datasources.") + return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err) } } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 767097f0e2e..0de6f037844 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -51,12 +51,15 @@ var ( ) type Flags struct { - ConfigFile string - TraceLevel bool - DebugLevel bool - InfoLevel bool - WarnLevel bool - ErrorLevel bool + ConfigFile string + + LogLevelTrace bool + LogLevelDebug bool + LogLevelInfo bool + LogLevelWarn bool + LogLevelError bool + LogLevelFatal bool + PrintVersion bool SingleFileType string Labels map[string]string @@ -107,7 +110,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error { dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform) if err != nil { - return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN) + return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err) } } else { dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec) @@ -116,6 +119,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error { } } + if len(dataSources) == 0 { + return fmt.Errorf("no datasource enabled") + } + return nil } @@ -140,11 +147,14 @@ func (l labelsMap) Set(label string) error { func (f *Flags) Parse() { flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file") - flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose") - flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr") - flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr") - flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr") - flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr") + + flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)") + flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'") + flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'") + flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'") + flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'") + flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'") + flag.BoolVar(&f.PrintVersion, "version", false, "display version") flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine") flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition") @@ -172,16 +182,18 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { // override from flags switch { - case f.TraceLevel: + case f.LogLevelTrace: ret = log.TraceLevel - case f.DebugLevel: + case f.LogLevelDebug: ret = log.DebugLevel - case f.InfoLevel: + case f.LogLevelInfo: ret = log.InfoLevel - case f.WarnLevel: + case f.LogLevelWarn: ret = log.WarnLevel - case f.ErrorLevel: + case f.LogLevelError: ret = log.ErrorLevel + case f.LogLevelFatal: + ret = log.FatalLevel default: } diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 17cc99827de..67489f459ff 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -7,6 +7,10 @@ import ( "sync" "time" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -16,9 +20,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) { @@ -50,11 +51,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error alertsToPush, err := dedupAlerts(alerts) if err != nil { - return errors.Wrap(err, "failed to transform alerts for api") + return fmt.Errorf("failed to transform alerts for api: %w", err) } _, _, err = client.Alerts.Add(ctx, alertsToPush) if err != nil { - return errors.Wrap(err, "failed sending alert to LAPI") + return fmt.Errorf("failed sending alert to LAPI: %w", err) } return nil } @@ -104,11 +105,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky Scenarios: scenarios, }) if err != nil { - return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login) + return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err) } if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { - return errors.Wrap(err, "unable to parse jwt expiration") + return fmt.Errorf("unable to parse jwt expiration: %w", err) } Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index adb072376f9..3f717e3975d 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -12,9 +12,7 @@ import ( ) func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error { - var ( - count int - ) + count := 0 for { //bucket is now ready select { diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index c51d24147cd..d63a587ac16 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows/svc" @@ -22,7 +21,7 @@ func StartRunSvc() error { isRunninginService, err := svc.IsWindowsService() if err != nil { - return errors.Wrap(err, "failed to determine if we are running in windows service mode") + return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err) } if isRunninginService { return runService(svcName) @@ -31,22 +30,22 @@ func StartRunSvc() error { if flags.WinSvc == "Install" { err = installService(svcName, svcDescription) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Remove" { err = removeService(svcName) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Start" { err = startService(svcName) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Stop" { err = controlService(svcName, svc.Stop, svc.Stopped) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "" { return WindowsRun() @@ -66,7 +65,7 @@ func WindowsRun() error { if err != nil { return err } - // Configure logging + log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 5e2e8b720a3..5d365b410d7 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -1,16 +1,16 @@ package main import ( + "fmt" "os" "os/signal" "syscall" "time" - "github.com/coreos/go-systemd/v22/daemon" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/csdaemon" "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } apiServer, err := initAPIServer(cConfig) if err != nil { - return nil, errors.Wrap(err, "unable to init api server") + return nil, fmt.Errorf("unable to init api server: %w", err) } apiReady := make(chan bool, 1) @@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return nil, errors.Wrap(err, "unable to init crowdsec") + return nil, fmt.Errorf("unable to init crowdsec: %w", err) } // restore bucket state @@ -180,13 +180,13 @@ func shutdownCrowdsec() error { func shutdown(sig os.Signal, cConfig *csconfig.Config) error { if !cConfig.DisableAgent { if err := shutdownCrowdsec(); err != nil { - return errors.Wrap(err, "failed to shut down crowdsec") + return fmt.Errorf("failed to shut down crowdsec: %w", err) } } if !cConfig.DisableAPI { if err := shutdownAPI(); err != nil { - return errors.Wrap(err, "failed to shut down api routines") + return fmt.Errorf("failed to shut down api routines: %w", err) } } @@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error { log.Warning("SIGHUP received, reloading") if err = shutdown(s, cConfig); err != nil { - exitChan <- errors.Wrap(err, "failed shutdown") + exitChan <- fmt.Errorf("failed shutdown: %w", err) break Loop } if newConfig, err = reloadHandler(s); err != nil { - exitChan <- errors.Wrap(err, "reload handler failure") + exitChan <- fmt.Errorf("reload handler failure: %w", err) break Loop } @@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error { case os.Interrupt, syscall.SIGTERM: log.Warning("SIGTERM received, shutting down") if err = shutdown(s, cConfig); err != nil { - exitChan <- errors.Wrap(err, "failed shutdown") + exitChan <- fmt.Errorf("failed shutdown: %w", err) break Loop } @@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil { dbClient, err := database.NewClient(cConfig.API.Server.DbConfig) if err != nil { - return errors.Wrap(err, "failed to get database client") + return fmt.Errorf("failed to get database client: %w", err) } err = exprhelpers.Init(dbClient) if err != nil { - return errors.Wrap(err, "failed to init expr helpers") + return fmt.Errorf("failed to init expr helpers: %w", err) } } else { err := exprhelpers.Init(nil) if err != nil { - return errors.Wrap(err, "failed to init expr helpers") + return fmt.Errorf("failed to init expr helpers: %w", err) } log.Warningln("Exprhelpers loaded without database client.") @@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled { log.Infof("Crowdsec CTI helper enabled") if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil { - return errors.Wrap(err, "failed to init crowdsec cti") + return fmt.Errorf("failed to init crowdsec cti: %w", err) } } @@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e apiServer, err := initAPIServer(cConfig) if err != nil { - return errors.Wrap(err, "api server init") + return fmt.Errorf("api server init: %w", err) } if !flags.TestMode { @@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return errors.Wrap(err, "crowdsec init") + return fmt.Errorf("crowdsec init: %w", err) } // if it's just linting, we're done @@ -350,10 +350,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if cConfig.Common != nil && cConfig.Common.Daemonize { - sent, err := daemon.SdNotify(false, daemon.SdNotifyReady) - if !sent || err != nil { - log.Errorf("Failed to notify(sent: %v): %v", sent, err) - } + csdaemon.NotifySystemd(log.StandardLogger()) // wait for signals return HandleSignals(cConfig) } diff --git a/cmd/crowdsec/win_service.go b/cmd/crowdsec/win_service.go index d0e80c58a53..ab9ecc8151f 100644 --- a/cmd/crowdsec/win_service.go +++ b/cmd/crowdsec/win_service.go @@ -8,10 +8,10 @@ package main import ( + "fmt" "syscall" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" @@ -106,7 +106,7 @@ func runService(name string) error { winsvc := crowdsec_winservice{config: cConfig} if err := svc.Run(name, &winsvc); err != nil { - return errors.Wrapf(err, "%s service failed", name) + return fmt.Errorf("%s service failed: %w", name, err) } log.Infof("%s service stopped", name) diff --git a/debian/control b/debian/control index d06d3888484..4673284e7b4 100644 --- a/debian/control +++ b/debian/control @@ -1,6 +1,8 @@ Source: crowdsec Maintainer: Crowdsec Team -Build-Depends: debhelper, bash, git +Build-Depends: debhelper, bash +Section: admin +Priority: optional Package: crowdsec Architecture: any diff --git a/debian/rules b/debian/rules index 6683e5443f2..e6202a6f774 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f -export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ') export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on @@ -11,12 +11,10 @@ override_dh_auto_clean: override_dh_auto_test: override_dh_auto_build: override_dh_auto_install: -# mkdir /tmp/go -# echo $(go version) -# echo $($GOCMD version) -# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd .. -# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd .. - make build + + # just use the prebuilt binaries, otherwise: + # make build BUILD_RE_WASM=0 BUILD_STATIC=1 + mkdir -p debian/crowdsec/usr/bin mkdir -p debian/crowdsec/etc/crowdsec mkdir -p debian/crowdsec/usr/share/crowdsec diff --git a/debian/templates b/debian/templates index c07ef8446d4..c6998eb8535 100644 --- a/debian/templates +++ b/debian/templates @@ -17,7 +17,7 @@ Description: Address of the local API server Template: crowdsec/capi Type: boolean Default: true -Description: Do you want to the centralized remote API server ? +Description: Do you want to use the centralized remote API server ? To share information with other crowdsec you can register to the centralized remote API server. . If you don't know what to do, answer yes. diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 8ec44910311..21b42dcb0b4 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -56,7 +56,7 @@ conf_get() { if [ $# -ge 2 ]; then yq e "$1" "$2" else - yq e "$1" "$CONFIG_FILE" + cscli config show-yaml | yq e "$1" fi } diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index 81567954bc7..b890bebb9c6 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -6,11 +6,8 @@ from http import HTTPStatus import json -import os -import pwd import pytest -import yaml pytestmark = pytest.mark.docker @@ -85,12 +82,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): 'COLLECTIONS': f'{coll}' } - hub = tmp_path_factory.mktemp("hub") - volumes = { - hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'} - } - - with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: + with crowdsec(flavor=flavor, environment=env) as cs: cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run('cscli collections list -o json') assert res.exit_code == 0 @@ -102,25 +94,13 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): f'*Enabled collections : {coll}*', ]) - # change file permissions to allow edit - current_uid = pwd.getpwuid(os.getuid()).pw_uid - res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub') - assert res.exit_code == 0 - - scenario = 'crowdsecurity/http-crawl-non_statics' - scenario_file = hub / f'scenarios/{scenario}.yaml' + scenario = 'crowdsecurity/http-crawl-non_statics' - with open(scenario_file) as f: - yml = yaml.safe_load(f) - - yml['description'] += ' (tainted)' - # won't be able to read it back because description is taken from the index - - with open(scenario_file, 'w') as f: - yaml.dump(yml, f) + # the description won't be read back, it's from the index + yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml" + res = cs.cont.exec_run(yq_command) + assert res.exit_code == 0 - with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: - cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json') assert res.exit_code == 0 j = json.loads(res.output) diff --git a/go.mod b/go.mod index 272fbc37091..7232bda234e 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,22 @@ go 1.20 require ( entgo.io/ent v0.11.3 github.com/AlecAivazis/survey/v2 v2.2.7 - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Masterminds/semver/v3 v3.1.1 + github.com/Masterminds/sprig/v3 v3.2.2 github.com/alexliesenfeld/health v0.5.1 github.com/antonmedv/expr v1.12.5 github.com/appleboy/gin-jwt/v2 v2.8.0 + github.com/aquasecurity/table v1.8.0 + github.com/aws/aws-lambda-go v1.38.0 github.com/aws/aws-sdk-go v1.42.25 + github.com/beevik/etree v1.1.0 + github.com/blackfireio/osinfo v1.0.3 + github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 github.com/c-robinson/iplib v1.0.3 + github.com/cespare/xxhash/v2 v2.1.2 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 + github.com/crowdsecurity/go-cs-lib v0.0.2 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 @@ -20,7 +28,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.17.0 @@ -29,65 +37,54 @@ require ( github.com/go-openapi/swag v0.19.14 github.com/go-openapi/validate v0.20.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/goccy/go-yaml v1.9.7 + github.com/gofrs/uuid v4.0.0+incompatible + github.com/golang-jwt/jwt/v4 v4.2.0 github.com/google/go-querystring v1.0.0 github.com/google/uuid v1.3.0 + github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/hashicorp/go-hclog v1.0.0 github.com/hashicorp/go-plugin v1.4.2 github.com/hashicorp/go-version v1.2.1 + github.com/ivanpirog/coloredcobra v1.0.1 github.com/jackc/pgx/v4 v4.14.1 github.com/jarcoal/httpmock v1.1.0 github.com/jszwec/csvutil v1.5.1 + github.com/lithammer/dedent v1.1.0 + github.com/mattn/go-isatty v0.0.17 github.com/mattn/go-sqlite3 v1.14.16 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nxadm/tail v1.4.8 github.com/oschwald/geoip2-golang v1.4.0 github.com/oschwald/maxminddb-golang v1.8.0 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 + github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 + github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c + github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 + github.com/wasilibs/go-re2 v0.2.1 golang.org/x/crypto v0.11.0 - golang.org/x/mod v0.8.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/mod v0.11.0 + golang.org/x/sys v0.10.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 - gotest.tools/v3 v3.0.3 ) require ( - github.com/Masterminds/semver v1.5.0 - github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aquasecurity/table v1.8.0 - github.com/aws/aws-lambda-go v1.38.0 - github.com/beevik/etree v1.1.0 - github.com/blackfireio/osinfo v1.0.3 - github.com/bluele/gcache v0.0.2 - github.com/cespare/xxhash/v2 v2.1.2 - github.com/coreos/go-systemd/v22 v22.5.0 github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f - github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd - github.com/goccy/go-yaml v1.9.7 - github.com/gofrs/uuid v4.0.0+incompatible - github.com/golang-jwt/jwt/v4 v4.2.0 - github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b - github.com/ivanpirog/coloredcobra v1.0.1 - github.com/lithammer/dedent v1.1.0 - github.com/mattn/go-isatty v0.0.14 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 - github.com/segmentio/kafka-go v0.4.34 - github.com/shirou/gopsutil/v3 v3.22.12 - github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - github.com/wasilibs/go-re2 v0.2.1 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sys v0.10.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -95,7 +92,7 @@ require ( require ( ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/agext/levenshtein v1.2.1 // indirect @@ -104,6 +101,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/corazawaf/libinjection-go v0.1.2 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -147,7 +145,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect @@ -171,6 +169,7 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -182,13 +181,14 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zclconf/go-cty v1.8.0 // indirect - go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.12.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + go.mongodb.org/mongo-driver v1.9.4 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect diff --git a/go.sum b/go.sum index 6b64c39cfc9..3345241dbfa 100644 --- a/go.sum +++ b/go.sum @@ -55,14 +55,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= @@ -172,14 +170,12 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5 h1:zpU5j3EOvH/Hxs3trWwsXp7Vr1T0YP0MIdp89RvVlY8= -github.com/crowdsecurity/coraza/v3 v3.0.0-20230713124654-1e01677657b5/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f h1:7MgSs0ryJrdGV0f17xBQJPfdD8oo/mqcIQjpqdPo2aA= github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= -github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc= +github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps= +github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -217,8 +213,9 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -665,16 +662,18 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -762,8 +761,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= -github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e h1:POJco99aNgosh92lGqmx7L1ei+kCymivB/419SD15PQ= github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -842,8 +839,12 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -894,7 +895,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -949,8 +949,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -969,8 +969,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck= -go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc= +go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1025,8 +1025,6 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1039,8 +1037,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1063,8 +1061,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1117,8 +1115,6 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1138,8 +1134,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1211,25 +1207,21 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1241,8 +1233,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1313,6 +1303,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/mk/__gmsl b/mk/__gmsl new file mode 100644 index 00000000000..6cd4a3090bc --- /dev/null +++ b/mk/__gmsl @@ -0,0 +1,969 @@ +# ---------------------------------------------------------------------------- +# +# GNU Make Standard Library (GMSL) +# +# A library of functions to be used with GNU Make's $(call) that +# provides functionality not available in standard GNU Make. +# +# Copyright (c) 2005-2022 John Graham-Cumming +# +# This file is part of GMSL +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# Neither the name of the John Graham-Cumming nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# ---------------------------------------------------------------------------- + +# This is the GNU Make Standard Library version number as a list with +# three items: major, minor, revision + +gmsl_version := 1 2 0 + +__gmsl_name := GNU Make Standard Library + +# Used to output warnings and error from the library, it's possible to +# disable any warnings or errors by overriding these definitions +# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS + +ifdef GMSL_NO_WARNINGS +__gmsl_warning := +else +__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1)) +endif + +ifdef GMSL_NO_ERRORS +__gmsl_error := +else + __gmsl_error = $(if $1,$(error $(__gmsl_name): $1)) +endif + +# If GMSL_TRACE is enabled then calls to the library functions are +# traced to stdout using warning messages with their arguments + +ifdef GMSL_TRACE +__gmsl_tr1 = $(warning $0('$1')) +__gmsl_tr2 = $(warning $0('$1','$2')) +__gmsl_tr3 = $(warning $0('$1','$2','$3')) +else +__gmsl_tr1 := +__gmsl_tr2 := +__gmsl_tr3 := +endif + +# See if spaces are valid in variable names (this was the case until +# GNU Make 3.82) +ifeq ($(MAKE_VERSION),3.82) +__gmsl_spaced_vars := $(false) +else +__gmsl_spaced_vars := $(true) +endif + +# Figure out whether we have $(eval) or not (GNU Make 3.80 and above) +# if we do not then output a warning message, if we do then some +# functions will be enabled. + +__gmsl_have_eval := $(false) +__gmsl_ignore := $(eval __gmsl_have_eval := $(true)) + +# If this is being run with Electric Cloud's emake then warn that +# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0, +# 5.1, 5.2 and 5.3 + +ifdef ECLOUD_BUILD_ID +__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION))) +__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION))) +ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","") +$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support) +__gmsl_have_eval := $(false) +endif +endif + +# See if we have $(lastword) (GNU Make 3.81 and above) + +__gmsl_have_lastword := $(lastword $(false) $(true)) + +# See if we have native or and and (GNU Make 3.81 and above) + +__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/ +__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/ +__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false)) +__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false)) + +ifneq ($(__gmsl_have_eval),$(true)) +$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled) +endif + +__gmsl_dollar := $$ +__gmsl_hash := \# + +# ---------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- +# Function: gmsl_compatible +# Arguments: List containing the desired library version number (maj min rev) +# Returns: $(true) if this version of the library is compatible +# with the requested version number, otherwise $(false) +# ---------------------------------------------------------------------------- +gmsl_compatible = $(strip \ + $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \ + $(false), \ + $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \ + $(true), \ + $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \ + $(false), \ + $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \ + $(true), \ + $(call lte,$(word 3,$1),$(word 3,$(gmsl_version)))))))) + +# ########################################################################### +# LOGICAL OPERATORS +# ########################################################################### + +# not is defined in gmsl + +# ---------------------------------------------------------------------------- +# Function: and +# Arguments: Two boolean values +# Returns: Returns $(true) if both of the booleans are true +# ---------------------------------------------------------------------------- +ifneq ($(__gmsl_have_and),$(true)) +and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false)) +endif + +# ---------------------------------------------------------------------------- +# Function: or +# Arguments: Two boolean values +# Returns: Returns $(true) if either of the booleans is true +# ---------------------------------------------------------------------------- +ifneq ($(__gmsl_have_or),$(true)) +or = $(__gmsl_tr2)$(if $1$2,$(true),$(false)) +endif + +# ---------------------------------------------------------------------------- +# Function: xor +# Arguments: Two boolean values +# Returns: Returns $(true) if exactly one of the booleans is true +# ---------------------------------------------------------------------------- +xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false))) + +# ---------------------------------------------------------------------------- +# Function: nand +# Arguments: Two boolean values +# Returns: Returns value of 'not and' +# ---------------------------------------------------------------------------- +nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true)) + +# ---------------------------------------------------------------------------- +# Function: nor +# Arguments: Two boolean values +# Returns: Returns value of 'not or' +# ---------------------------------------------------------------------------- +nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: xnor +# Arguments: Two boolean values +# Returns: Returns value of 'not xor' +# ---------------------------------------------------------------------------- +xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true))) + +# ########################################################################### +# LIST MANIPULATION FUNCTIONS +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: first (same as LISP's car, or head) +# Arguments: 1: A list +# Returns: Returns the first element of a list +# ---------------------------------------------------------------------------- +first = $(__gmsl_tr1)$(firstword $1) + +# ---------------------------------------------------------------------------- +# Function: last +# Arguments: 1: A list +# Returns: Returns the last element of a list +# ---------------------------------------------------------------------------- +ifeq ($(__gmsl_have_lastword),$(true)) +last = $(__gmsl_tr1)$(lastword $1) +else +last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1)) +endif + +# ---------------------------------------------------------------------------- +# Function: rest (same as LISP's cdr, or tail) +# Arguments: 1: A list +# Returns: Returns the list with the first element removed +# ---------------------------------------------------------------------------- +rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1) + +# ---------------------------------------------------------------------------- +# Function: chop +# Arguments: 1: A list +# Returns: Returns the list with the last element removed +# ---------------------------------------------------------------------------- +chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1) + +# ---------------------------------------------------------------------------- +# Function: map +# Arguments: 1: Name of function to $(call) for each element of list +# 2: List to iterate over calling the function in 1 +# Returns: The list after calling the function on each element +# ---------------------------------------------------------------------------- +map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a))) + +# ---------------------------------------------------------------------------- +# Function: pairmap +# Arguments: 1: Name of function to $(call) for each pair of elements +# 2: List to iterate over calling the function in 1 +# 3: Second list to iterate over calling the function in 1 +# Returns: The list after calling the function on each pair of elements +# ---------------------------------------------------------------------------- +pairmap = $(strip $(__gmsl_tr3)\ + $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \ + $(call pairmap,$1,$(call rest,$2),$(call rest,$3)))) + +# ---------------------------------------------------------------------------- +# Function: leq +# Arguments: 1: A list to compare against... +# 2: ...this list +# Returns: Returns $(true) if the two lists are identical +# ---------------------------------------------------------------------------- +leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \ + $(call __gmsl_list_equal,$1,$2),$(false))) + +__gmsl_list_equal = $(if $(strip $1), \ + $(if $(call seq,$(call first,$1),$(call first,$2)), \ + $(call __gmsl_list_equal, \ + $(call rest,$1), \ + $(call rest,$2)), \ + $(false)), \ + $(true)) + +# ---------------------------------------------------------------------------- +# Function: lne +# Arguments: 1: A list to compare against... +# 2: ...this list +# Returns: Returns $(true) if the two lists are different +# ---------------------------------------------------------------------------- +lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: reverse +# Arguments: 1: A list to reverse +# Returns: The list with its elements in reverse order +# ---------------------------------------------------------------------------- +reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \ + $(call first,$1))) + +# ---------------------------------------------------------------------------- +# Function: uniq +# Arguments: 1: A list from which to remove repeated elements +# Returns: The list with duplicate elements removed without reordering +# ---------------------------------------------------------------------------- +uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \ + $(call uniq,$(filter-out $(firstword $1),$1)))) + +# ---------------------------------------------------------------------------- +# Function: length +# Arguments: 1: A list +# Returns: The number of elements in the list +# ---------------------------------------------------------------------------- +length = $(__gmsl_tr1)$(words $1) + +# ########################################################################### +# STRING MANIPULATION FUNCTIONS +# ########################################################################### + +# Helper function that translates any GNU Make 'true' value (i.e. a +# non-empty string) to our $(true) + +__gmsl_make_bool = $(if $(strip $1),$(true),$(false)) + +# ---------------------------------------------------------------------------- +# Function: seq +# Arguments: 1: A string to compare against... +# 2: ...this string +# Returns: Returns $(true) if the two strings are identical +# ---------------------------------------------------------------------------- +seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: sne +# Arguments: 1: A string to compare against... +# 2: ...this string +# Returns: Returns $(true) if the two strings are not the same +# ---------------------------------------------------------------------------- +sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: split +# Arguments: 1: The character to split on +# 2: A string to split +# Returns: Splits a string into a list separated by spaces at the split +# character in the first argument +# ---------------------------------------------------------------------------- +split = $(__gmsl_tr2)$(strip $(subst $1, ,$2)) + +# ---------------------------------------------------------------------------- +# Function: merge +# Arguments: 1: The character to put between fields +# 2: A list to merge into a string +# Returns: Merges a list into a single string, list elements are separated +# by the character in the first argument +# ---------------------------------------------------------------------------- +merge = $(__gmsl_tr2)$(strip $(if $2, \ + $(if $(call seq,1,$(words $2)), \ + $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2))))) + +ifdef __gmsl_have_eval +# ---------------------------------------------------------------------------- +# Function: tr +# Arguments: 1: The list of characters to translate from +# 2: The list of characters to translate to +# 3: The text to translate +# Returns: Returns the text after translating characters +# ---------------------------------------------------------------------------- +tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \ + $(eval __gmsl_t := $3) \ + $(foreach c, \ + $(join $(addsuffix :,$1),$2), \ + $(eval __gmsl_t := \ + $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \ + $(__gmsl_t))))$(__gmsl_t)) + +# Common character classes for use with the tr function. Each of +# these is actually a variable declaration and must be wrapped with +# $() or ${} to be used. + +[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z # +[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z # +[0-9] := 0 1 2 3 4 5 6 7 8 9 # +[A-F] := A B C D E F # + +# ---------------------------------------------------------------------------- +# Function: uc +# Arguments: 1: Text to upper case +# Returns: Returns the text in upper case +# ---------------------------------------------------------------------------- +uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1) + +# ---------------------------------------------------------------------------- +# Function: lc +# Arguments: 1: Text to lower case +# Returns: Returns the text in lower case +# ---------------------------------------------------------------------------- +lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1) + +# ---------------------------------------------------------------------------- +# Function: strlen +# Arguments: 1: A string +# Returns: Returns the length of the string +# ---------------------------------------------------------------------------- + +# This results in __gmsl_tab containing a tab + +__gmsl_tab := # + +__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z +__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z +__gmsl_characters += 0 1 2 3 4 5 6 7 8 9 +__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = + +__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? | + +# This results in __gmsl_space containing just a space + +__gmsl_empty := +__gmsl_space := $(__gmsl_empty) $(__gmsl_empty) + +strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp))) + +# This results in __gmsl_newline containing just a newline + +define __gmsl_newline + + +endef + +# ---------------------------------------------------------------------------- +# Function: substr +# Arguments: 1: A string +# 2: Start position (first character is 1) +# 3: End position (inclusive) +# Returns: A substring. +# Note: The string in $1 must not contain a § +# ---------------------------------------------------------------------------- + +substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp)))) + +endif # __gmsl_have_eval + +# ########################################################################### +# SET MANIPULATION FUNCTIONS +# ########################################################################### + +# Sets are represented by sorted, deduplicated lists. To create a set +# from a list use set_create, or start with the empty_set and +# set_insert individual elements + +# This is the empty set +empty_set := + +# ---------------------------------------------------------------------------- +# Function: set_create +# Arguments: 1: A list of set elements +# Returns: Returns the newly created set +# ---------------------------------------------------------------------------- +set_create = $(__gmsl_tr1)$(sort $1) + +# ---------------------------------------------------------------------------- +# Function: set_insert +# Arguments: 1: A single element to add to a set +# 2: A set +# Returns: Returns the set with the element added +# ---------------------------------------------------------------------------- +set_insert = $(__gmsl_tr2)$(sort $1 $2) + +# ---------------------------------------------------------------------------- +# Function: set_remove +# Arguments: 1: A single element to remove from a set +# 2: A set +# Returns: Returns the set with the element removed +# ---------------------------------------------------------------------------- +set_remove = $(__gmsl_tr2)$(filter-out $1,$2) + +# ---------------------------------------------------------------------------- +# Function: set_is_member, set_is_not_member +# Arguments: 1: A single element +# 2: A set +# Returns: (set_is_member) Returns $(true) if the element is in the set +# (set_is_not_member) Returns $(false) if the element is in the set +# ---------------------------------------------------------------------------- +set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false)) +set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: set_union +# Arguments: 1: A set +# 2: Another set +# Returns: Returns the union of the two sets +# ---------------------------------------------------------------------------- +set_union = $(__gmsl_tr2)$(sort $1 $2) + +# ---------------------------------------------------------------------------- +# Function: set_intersection +# Arguments: 1: A set +# 2: Another set +# Returns: Returns the intersection of the two sets +# ---------------------------------------------------------------------------- +set_intersection = $(__gmsl_tr2)$(filter $1,$2) + +# ---------------------------------------------------------------------------- +# Function: set_is_subset +# Arguments: 1: A set +# 2: Another set +# Returns: Returns $(true) if the first set is a subset of the second +# ---------------------------------------------------------------------------- +set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1) + +# ---------------------------------------------------------------------------- +# Function: set_equal +# Arguments: 1: A set +# 2: Another set +# Returns: Returns $(true) if the two sets are identical +# ---------------------------------------------------------------------------- +set_equal = $(__gmsl_tr2)$(call seq,$1,$2) + +# ########################################################################### +# ARITHMETIC LIBRARY +# ########################################################################### + +# Integers a represented by lists with the equivalent number of x's. +# For example the number 4 is x x x x. + +# ---------------------------------------------------------------------------- +# Function: int_decode +# Arguments: 1: A number of x's representation +# Returns: Returns the integer for human consumption that is represented +# by the string of x's +# ---------------------------------------------------------------------------- +int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0) + +# ---------------------------------------------------------------------------- +# Function: int_encode +# Arguments: 1: A number in human-readable integer form +# Returns: Returns the integer encoded as a string of x's +# ---------------------------------------------------------------------------- +__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x)))) +__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0) +int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1)) + +# The arithmetic library functions come in two forms: one form of each +# function takes integers as arguments and the other form takes the +# encoded form (x's created by a call to int_encode). For example, +# there are two plus functions: +# +# plus Called with integer arguments and returns an integer +# int_plus Called with encoded arguments and returns an encoded result +# +# plus will be slower than int_plus because its arguments and result +# have to be translated between the x's format and integers. If doing +# a complex calculation use the int_* forms with a single encoding of +# inputs and single decoding of the output. For simple calculations +# the direct forms can be used. + +# Helper function used to wrap an int_* function into a function that +# takes a pair of integers, perhaps a function and returns an integer +# result +__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3))) +__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2))) +__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3)) + +# ---------------------------------------------------------------------------- +# Function: int_plus +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the sum of the two numbers in x's representation +# ---------------------------------------------------------------------------- +int_plus = $(strip $(__gmsl_tr2)$1 $2) + +# ---------------------------------------------------------------------------- +# Function: plus (wrapped version of int_plus) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the sum of the two integers +# ---------------------------------------------------------------------------- +plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_subtract +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the difference of the two numbers in x's representation, +# or outputs an error on a numeric underflow +# ---------------------------------------------------------------------------- +int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \ + $(filter-out xx,$(join $1,$2)), \ + $(call __gmsl_warning,Subtraction underflow))) + +# ---------------------------------------------------------------------------- +# Function: subtract (wrapped version of int_subtract) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the difference of the two integers, +# or outputs an error on a numeric underflow +# ---------------------------------------------------------------------------- +subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_multiply +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the product of the two numbers in x's representation +# ---------------------------------------------------------------------------- +int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: multiply (wrapped version of int_multiply) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the product of the two integers +# ---------------------------------------------------------------------------- +multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_divide +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the result of integer division of argument 1 divided +# by argument 2 in x's representation +# ---------------------------------------------------------------------------- +int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ + $(subst M,x,$(filter-out x,$(subst $2,M,$1))), \ + $(call __gmsl_error,Division by zero)))) + +# ---------------------------------------------------------------------------- +# Function: divide (wrapped version of int_divide) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the integer division of the first argument by the second +# ---------------------------------------------------------------------------- +divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_modulo +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the remainder of integer division of argument 1 divided +# by argument 2 in x's representation +# ---------------------------------------------------------------------------- +int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ + $(filter-out M,$(subst $2,M,$1)), \ + $(call __gmsl_error,Division by zero)))) + +# ---------------------------------------------------------------------------- +# Function: modulo (wrapped version of int_modulo) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the remainder of integer division of the first argument +# by the second +# ---------------------------------------------------------------------------- +modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_max, int_min +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the maximum or minimum of its arguments in x's +# representation +# ---------------------------------------------------------------------------- +int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2)) +int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2))) + +# ---------------------------------------------------------------------------- +# Function: max, min +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the maximum or minimum of its integer arguments +# ---------------------------------------------------------------------------- +max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2) +min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne +# Arguments: Two x's representation numbers to be compared +# Returns: $(true) or $(false) +# +# int_gt First argument greater than second argument +# int_gte First argument greater than or equal to second argument +# int_lt First argument less than second argument +# int_lte First argument less than or equal to second argument +# int_eq First argument is numerically equal to the second argument +# int_ne First argument is not numerically equal to the second argument +# ---------------------------------------------------------------------------- +int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $2), \ + $(words $(call int_max,$1,$2)))) +int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(call int_gt,$1,$2)$(call int_eq,$1,$2)) +int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $1), \ + $(words $(call int_max,$1,$2)))) +int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(call int_lt,$1,$2)$(call int_eq,$1,$2)) +int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter $(words $1),$(words $2))) +int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $1),$(words $2))) + +# ---------------------------------------------------------------------------- +# Function: gt, gte, lt, lte, eq, ne +# Arguments: Two integers to be compared +# Returns: $(true) or $(false) +# +# gt First argument greater than second argument +# gte First argument greater than or equal to second argument +# lt First argument less than second argument +# lte First argument less than or equal to second argument +# eq First argument is numerically equal to the second argument +# ne First argument is not numerically equal to the second argument +# ---------------------------------------------------------------------------- +gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2) +gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2) +lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2) +lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2) +eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2) +ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2) + +# increment adds 1 to its argument, decrement subtracts 1. Note that +# decrement does not range check and hence will not underflow, but +# will incorrectly say that 0 - 1 = 0 + +# ---------------------------------------------------------------------------- +# Function: int_inc +# Arguments: 1: A number in x's representation +# Returns: The number incremented by 1 in x's representation +# ---------------------------------------------------------------------------- +int_inc = $(strip $(__gmsl_tr1)$1 x) + +# ---------------------------------------------------------------------------- +# Function: inc +# Arguments: 1: An integer +# Returns: The argument incremented by 1 +# ---------------------------------------------------------------------------- +inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1) + +# ---------------------------------------------------------------------------- +# Function: int_dec +# Arguments: 1: A number in x's representation +# Returns: The number decremented by 1 in x's representation +# ---------------------------------------------------------------------------- +int_dec = $(__gmsl_tr1)$(strip \ + $(if $(call sne,0,$(words $1)), \ + $(wordlist 2,$(words $1),$1))) + +# ---------------------------------------------------------------------------- +# Function: dec +# Arguments: 1: An integer +# Returns: The argument decremented by 1 +# ---------------------------------------------------------------------------- +dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1) + +# double doubles its argument, and halve halves it + +# ---------------------------------------------------------------------------- +# Function: int_double +# Arguments: 1: A number in x's representation +# Returns: The number doubled (i.e. * 2) and returned in x's representation +# ---------------------------------------------------------------------------- +int_double = $(strip $(__gmsl_tr1)$1 $1) + +# ---------------------------------------------------------------------------- +# Function: double +# Arguments: 1: An integer +# Returns: The integer times 2 +# ---------------------------------------------------------------------------- +double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1) + +# ---------------------------------------------------------------------------- +# Function: int_halve +# Arguments: 1: A number in x's representation +# Returns: The number halved (i.e. / 2) and returned in x's representation +# ---------------------------------------------------------------------------- +int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \ + $(join $1,$(foreach a,$1,y x))))) + +# ---------------------------------------------------------------------------- +# Function: halve +# Arguments: 1: An integer +# Returns: The integer divided by 2 +# ---------------------------------------------------------------------------- +halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1) + +# ---------------------------------------------------------------------------- +# Function: sequence +# Arguments: 1: An integer +# 2: An integer +# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or +# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1] +# ---------------------------------------------------------------------------- +sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \ + $(call __gmsl_sequence_up,$1,$2), \ + $(call __gmsl_sequence_dn,$2,$1))) + +__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2)) +__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2))) + +# ---------------------------------------------------------------------------- +# Function: dec2hex, dec2bin, dec2oct +# Arguments: 1: An integer +# Returns: The decimal argument converted to hexadecimal, binary or +# octal +# ---------------------------------------------------------------------------- + +__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1)))))) + +dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16)) +dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2)) +dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8)) + +__gmsl_base_divide = $(subst $2,X ,$1) +__gmsl_q = $(strip $(filter X,$1)) +__gmsl_r = $(words $(filter x,$1)) + +__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2) +__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2) + +ifdef __gmsl_have_eval +# ########################################################################### +# ASSOCIATIVE ARRAYS +# ########################################################################### + +# Magic string that is very unlikely to appear in a key or value + +__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6 + +# ---------------------------------------------------------------------------- +# Function: set +# Arguments: 1: Name of associative array +# 2: The key value to associate +# 3: The value associated with the key +# Returns: Nothing +# ---------------------------------------------------------------------------- +set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3) + +# Only used internally by memoize function + +__gmsl_set = $(call set,$1,$2,$3)$3 + +# ---------------------------------------------------------------------------- +# Function: get +# Arguments: 1: Name of associative array +# 2: The key to retrieve +# Returns: The value stored in the array for that key +# ---------------------------------------------------------------------------- +get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1)) + +# ---------------------------------------------------------------------------- +# Function: keys +# Arguments: 1: Name of associative array +# Returns: Returns a list of all defined keys in the array +# ---------------------------------------------------------------------------- +keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \ + $(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES)))) + +# ---------------------------------------------------------------------------- +# Function: defined +# Arguments: 1: Name of associative array +# 2: The key to test +# Returns: Returns true if the key is defined (i.e. not empty) +# ---------------------------------------------------------------------------- +defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),) + +endif # __gmsl_have_eval + +ifdef __gmsl_have_eval +# ########################################################################### +# NAMED STACKS +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: push +# Arguments: 1: Name of stack +# 2: Value to push onto the top of the stack (must not contain +# a space) +# Returns: None +# ---------------------------------------------------------------------------- +push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\ + $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1))) + +# ---------------------------------------------------------------------------- +# Function: pop +# Arguments: 1: Name of stack +# Returns: Top element from the stack after removing it +# ---------------------------------------------------------------------------- +pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \ + $(call first,$(__gmsl_stack_$1)) \ + $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1))))) + +# ---------------------------------------------------------------------------- +# Function: peek +# Arguments: 1: Name of stack +# Returns: Top element from the stack without removing it +# ---------------------------------------------------------------------------- +peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1)) + +# ---------------------------------------------------------------------------- +# Function: depth +# Arguments: 1: Name of stack +# Returns: Number of items on the stack +# ---------------------------------------------------------------------------- +depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1)) + +endif # __gmsl_have_eval + +ifdef __gmsl_have_eval +# ########################################################################### +# STRING CACHE +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: memoize +# Arguments: 1. Name of the function to be called if the string +# has not been previously seen +# 2. A string +# Returns: Returns the result of a memo function (which the user must +# define) on the passed in string and remembers the result. +# +# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache +# of MD5 hashes of strings. $(call memoize,memo,foo bar baz) +# ---------------------------------------------------------------------------- +__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2) +memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\ + $(call get,__gmsl_m,$(__gmsl_memoize)), \ + $(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2)))) + +endif # __gmsl_have_eval + +# ########################################################################### +# DEBUGGING FACILITIES +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Target: gmsl-echo-% +# Arguments: The % should be replaced by the name of a variable that you +# wish to print out. +# Action: Echos the value of the variable that matches the %. +# For example, 'make gmsl-echo-SHELL' will output the value of +# the SHELL variable. +# ---------------------------------------------------------------------------- +gmsl-echo-%: ; @echo $($*) + +# ---------------------------------------------------------------------------- +# Target: gmsl-print-% +# Arguments: The % should be replaced by the name of a variable that you +# wish to print out. +# Action: Echos the name of the variable that matches the % and its value. +# For example, 'make gmsl-print-SHELL' will output the value of +# the SHELL variable +# ---------------------------------------------------------------------------- +gmsl-print-%: ; @echo $* = $($*) + +# ---------------------------------------------------------------------------- +# Function: assert +# Arguments: 1: A boolean that must be true or the assertion will fail +# 2: The message to print with the assertion +# Returns: None +# ---------------------------------------------------------------------------- +assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2))) + +# ---------------------------------------------------------------------------- +# Function: assert_exists +# Arguments: 1: Name of file that must exist, if it is missing an assertion +# will be generated +# Returns: None +# ---------------------------------------------------------------------------- +assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing)) + +# ---------------------------------------------------------------------------- +# Function: assert_no_dollar +# Arguments: 1: Name of a function being executd +# 2: Arguments to check +# Returns: None +# ---------------------------------------------------------------------------- +assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument) + +# ---------------------------------------------------------------------------- +# Function: assert_no_space +# Arguments: 1: Name of a function being executd +# 2: Arguments to check +# Returns: None +# ---------------------------------------------------------------------------- +ifeq ($(__gmsl_spaced_vars),$(false)) +assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument) +else +assert_no_space = +endif diff --git a/mk/gmsl b/mk/gmsl new file mode 100644 index 00000000000..b22949a31cf --- /dev/null +++ b/mk/gmsl @@ -0,0 +1,85 @@ +# ---------------------------------------------------------------------------- +# +# GNU Make Standard Library (GMSL) +# +# A library of functions to be used with GNU Make's $(call) that +# provides functionality not available in standard GNU Make. +# +# Copyright (c) 2005-2022 John Graham-Cumming +# +# This file is part of GMSL +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# Neither the name of the John Graham-Cumming nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# ---------------------------------------------------------------------------- + +# Determine if the library has already been included and if so don't +# bother including it again + +ifndef __gmsl_included + +# Standard definitions for true and false. true is any non-empty +# string, false is an empty string. These are intended for use with +# $(if). + +true := T +false := + +# ---------------------------------------------------------------------------- +# Function: not +# Arguments: 1: A boolean value +# Returns: Returns the opposite of the arg. (true -> false, false -> true) +# ---------------------------------------------------------------------------- +not = $(if $1,$(false),$(true)) + +# Prevent reinclusion of the library + +__gmsl_included := $(true) + +# Try to determine where this file is located. If the caller did +# include /foo/gmsl then extract the /foo/ so that __gmsl gets +# included transparently + +__gmsl_root := + +ifneq ($(MAKEFILE_LIST),) +__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) + +# If there are any spaces in the path in __gmsl_root then give up + +ifeq (1,$(words $(__gmsl_root))) +__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root)) +endif + +endif + +include $(__gmsl_root)__gmsl + +endif # __gmsl_included + diff --git a/mk/gmsl.html b/mk/gmsl.html new file mode 100644 index 00000000000..a8da4698752 --- /dev/null +++ b/mk/gmsl.html @@ -0,0 +1,733 @@ + + + + GNU Make Standard Library + + +

GNU Make Standard Library

+The GNU Make Standard Library (GMSL) is a collection of functions +implemented using native GNU Make functionality that provide list and +string manipulation, integer arithmetic, associative arrays, stacks, +and debugging facilities.  The GMSL is released under the BSD License.
+
+[Project Page] +[Releases] +
+

Using GMSL

+The two files needed are gmsl +and __gmsl.  To +include the GMSL in your Makefile do
+
include gmsl
+gmsl automatically includes __gmsl.  To check that +you have the right version of gmsl +use the gmsl_compatible +function (see +below). The current version is 1 2 0.
+
+The GMSL package also includes a test suite for GMSL.  Just run make -f gmsl-tests.
+

Logical Operators

GMSL has boolean $(true) (a non-empty string) +and $(false) (an empty string).  The following operators can be +used with those variables.
+
+
not
+ +
+ +Arguments: A boolean value
+ +Returns:   Returns $(true) if the boolean is $(false) and vice versa
+ +
and
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if both of the booleans are true
+
or
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if either of the booleans is true
+
xor
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if exactly one of the booleans is true
+
nand
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not and'
+
nor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not or'
+
xnor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not xor'
+
+

List Manipulation Functions

+ A list is a string of characters; the list separator is a space.
+ +
+
first
+
+Arguments: 1: A list
+Returns:   Returns the first element of a list
+ +
last
+
+Arguments: 1: A list
+Returns:   Returns the last element of a list
+ +
rest
+
+Arguments: 1: A list
+Returns:   Returns the list with the first element +removed
+ +
chop
+
+Arguments: 1: A list
+Returns:   Returns the list with the last element removed
+ +
map
+
+Arguments: 1: Name of function to +$(call) for each element of list
+           2: List to +iterate over calling the function in 1
+Returns:   The list after calling the function on each +element
+ +
pairmap
+
+Arguments: 1: Name of function to +$(call) for each pair of elements
+           2: List to +iterate over calling the function in 1
+           3: Second +list to iterate over calling the function in 1
+Returns:   The list after calling the function on each +pair of elements
+ +
leq
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are identical
+ +
lne
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are different
+ +
reverse
+
+Arguments: 1: A list to reverse
+Returns:   The list with its elements in reverse order
+ +
uniq
+
+Arguments: 1: A list to deduplicate
+Returns:   The list with elements in order without duplicates
+ +
length
+
+Arguments: 1: A list
+Returns:   The number of elements in the list
+ +
+

String Manipulation Functions

+A string is any sequence of characters.
+
+
seq
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are +identical
+ +
sne
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are not +the same
+ +
strlen
+
+Arguments: 1: A string
+Returns:   Returns the length of the string
+ +
substr
+
+Arguments: 1: A string
+           2: Start offset (first character is 1)
+           3: Ending offset (inclusive)
Returns:   Returns a substring
+ +
split
+
+Arguments: 1: The character to +split on
+           2: A +string to split
+Returns:   Splits a string into a list separated by +spaces at the split
+           character +in the first argument
+ +
merge
+
+Arguments: 1: The character to +put between fields
+           2: A list +to merge into a string
+Returns:   Merges a list into a single string, list +elements are separated
+           by the +character in the first argument
+ +
tr
+
+Arguments: 1: The list of +characters to translate from
+           2: The +list of characters to translate to
+           3: The +text to translate
+Returns:   Returns the text after translating characters
+ +
uc
+
+Arguments: 1: Text to upper case
+Returns:   Returns the text in upper case
+ +
lc
+
+Arguments: 1: Text to lower case
+Returns:   Returns the text in lower case
+ +
+

Set Manipulation Functions

+Sets are represented by sorted, deduplicated lists. To create a set +from a list use set_create, or start with the empty_set and set_insert individual elements. +The empty set is defined as empty_set.

+ +

set_create
+
+Arguments: 1: A list of set elements
+Returns:   Returns the newly created set
+ + +
set_insert
+
+Arguments: 1: A single element to add to a set
+           2: A set
+Returns:   Returns the set with the element added
+ + +
set_remove
+
+Arguments: 1: A single element to remove from a set
+           2: A set
+Returns:   Returns the set with the element removed
+ + +
set_is_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(true) if the element is in the set
+ + +
set_is_not_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(false) if the element is in the set
+ + +
set_union
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the union of the two sets
+ + +
set_intersection
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the intersection of the two sets
+ + +
set_is_subset
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the first set is a subset of the second
+ + +
set_equal
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the two sets are identical
+ + +
+

Integer Arithmetic Functions

+Integers are represented by lists with the equivalent number of +x's.  For example the number 4 is x x x x.  The maximum +integer that the library can handle as input (i.e. as the argument to a +call to int_encode) is +65536. There is no limit on integer size for internal computations or +output.
+
+The arithmetic library functions come in two forms: one form of each +function takes integers as arguments and the other form takes the +encoded form (x's created by a call to int_encode).  For example, +there are two plus functions: plus +(called with integer arguments and returns an integer) and int_plus (called with encoded +arguments and returns an encoded result).
+
+plus will be slower than int_plus because its arguments +and result have to be translated between the x's format and +integers.  If doing a complex calculation use the int_* forms with a single +encoding of inputs and single decoding of the output.  For simple +calculations the direct forms can be used.
+
+
int_decode
+
+Arguments: 1: A number of x's +representation
+Returns:   Returns the integer for human consumption +that is represented
+           by the +string of x's
+ +
int_encode
+
+Arguments: 1: A number in +human-readable integer form
+Returns:   Returns the integer encoded as a string of x's
+ +
int_plus
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the sum of the two numbers in x's +representation
+ +
plus (wrapped version of int_plus)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the sum of the two integers
+ +
int_subtract
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the difference of the two numbers in +x's representation,
+           or outputs +an error on a numeric underflow
+ +
subtract (wrapped version of int_subtract)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the difference of the two integers,
+           or outputs +an error on a numeric underflow
+ +
int_multiply
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the product of the two numbers in x's +representation
+ +
multiply (wrapped version of int_multiply)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the product of the two integers
+ +
int_divide
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the result of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
divide (wrapped version of int_divide)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the integer division of the first +argument by the second
+ +
int_modulo
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the remainder of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
modulo (wrapped version of int_modulo)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the remainder of integer division of the first +argument by the second
+ +
int_max, int_min
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the maximum or minimum of its +arguments in x's
+           +representation
+ +
max, min
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the maximum or minimum of its integer +arguments
+ +
int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
+
+Arguments: Two x's representation +numbers to be compared
+Returns:   $(true) or $(false)
+
+int_gt First argument greater than second argument
+int_gte First argument greater than or equal to second argument
+int_lt First argument less than second argument
+int_lte First argument less than or equal to second argument
+int_eq First argument is numerically equal to the second argument
+int_ne First argument is not numerically equal to the second argument
+ +
gt, gte, lt, lte, eq, ne
+
+Arguments: Two integers to be +compared
+Returns:   $(true) or $(false)
+
+gt First argument greater than second argument
+gte First argument greater than or equal to second argument
+lt First argument less than second argument
+lte First argument less than or equal to second argument
+eq First argument is numerically equal to the second argument
+ne First argument is not numerically equal to the second argument
+ +increment adds 1 to its argument, decrement subtracts 1. Note that
+decrement does not range check and hence will not underflow, but
+will incorrectly say that 0 - 1 = 0
+
int_inc
+
+Arguments: 1: A number in x's +representation
+Returns:   The number incremented by 1 in x's +representation
+ +
inc
+
+Arguments: 1: An integer
+Returns:   The argument incremented by 1
+ +
int_dec
+
+Arguments: 1: A number in x's +representation
+Returns:   The number decremented by 1 in x's +representation
+ +
dec
+
+Arguments: 1: An integer
+Returns:   The argument decremented by 1
+ +
int_double
+
+Arguments: 1: A number in x's +representation
+Returns:   The number doubled (i.e. * 2) and returned in +x's representation
+ +
double
+
+Arguments: 1: An integer
+Returns:   The integer times 2
+ +
int_halve
+
+Arguments: 1: A number in x's +representation
+Returns:   The number halved (i.e. / 2) and returned in +x's representation
+ +
halve
+
+Arguments: 1: An integer
+Returns:   The integer divided by 2
+ +
sequence
+
+Arguments: 1: An integer
+           2: An integer
+Returns:   The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1
+ +
dec2hex, dec2bin, dec2oct
+
+Arguments: 1: An integer
+Returns:   The decimal argument converted to hexadecimal, binary or octal
+ +
+

Associative Arrays

+An associate array maps a key value (a string with no spaces in it) to +a single value (any string).   
+
+ +
set
+
+Arguments: 1: Name of associative +array
+           2: The key +value to associate
+           3: The +value associated with the key
+Returns:   Nothing
+ +
get
+
+Arguments: 1: Name of associative +array
+           2: The key +to retrieve
+Returns:   The value stored in the array for that key
+ +
keys
+
+Arguments: 1: Name of associative +array
+Returns:   Returns a list of all defined keys in the +array
+ +
defined
+
+Arguments: 1: Name of associative +array
+           2: The key +to test
+Returns:   Returns true if the key is defined (i.e. not +empty)
+ +
+

Named Stacks

+A stack is an ordered list of strings (with no spaces in them).
+
+
push
+
+Arguments: 1: Name of stack
+           2: Value +to push onto the top of the stack (must not contain
+           a space)
+Returns:   None
+ +
pop
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack after removing it
+ +
peek
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack without removing it
+ +
depth
+
+Arguments: 1: Name of stack
+Returns:   Number of items on the stack
+ +
+

Function memoization

+To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.
+
+
memoize
+
+Arguments: 1: Name of function to memoize
+           2: String argument for the function
+Returns:   Result of $1 applied to $2 but only calls $1 once for each unique $2
+ + +
+

Miscellaneous and Debugging Facilities

+GMSL defines the following constants; all are accessed as normal GNU +Make variables by wrapping them in $() or ${}.
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Constant
+ 		Value
+ 		Purpose
+
true
+ 		T
+ 		Boolean for $(if) +and return from  GMSL functions
+
false
+
+ 		Boolean for $(if) +and return from GMSL functions
+
gmsl_version
+ 		1 0 0
+ 		GMSL version number as list: major minor revision
+
+
+gmsl_compatible
+
+Arguments: List containing the desired library version number (maj min +rev)
+Returns:   +$(true) if this version of the library is compatible
+           +with the requested version number, otherwise $(false) +
gmsl-print-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the name of the variable that matches +the % and its value.
+           For +example, 'make gmsl-print-SHELL' will output the value of
+           the SHELL +variable
+ +
gmsl-echo-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the value of the variable that matches +the %.
+           For +example, 'make gmsl-echo-SHELL' will output the value of
+           the SHELL +variable
+ +
assert
+
+Arguments: 1: A boolean that must +be true or the assertion will fail
+           2: The +message to print with the assertion
+Returns:   None
+ +
assert_exists
+
+Arguments: 1: Name of file that +must exist, if it is missing an assertion
+           will be +generated
+Returns:   None
+ +

+GMSL has a number of environment variables (or command-line overrides) +that control various bits of functionality:
+
+ + + + + + + + + + + + + + + + + + + +
Variable
+ 		Purpose
+
GMSL_NO_WARNINGS
+ 		If set prevents GMSL from outputting warning messages: +artithmetic functions generate underflow warnings.
+
GMSL_NO_ERRORS
+ 		If set prevents GMSL from generating fatal errors: division +by zero or failed assertions are fatal.
+
GMSL_TRACE
+ 		Enables function tracing.  Calls to GMSL functions will +result in name and arguments being traced.
+
+
+
+Copyright (c) 2005-2022 John Graham-Cumming.
+
+ diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index f611693f48b..8f06c93284f 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -7,8 +7,14 @@ MKDIR=mkdir -p # Go should not be required to run functional tests GOOS ?= $(shell go env GOOS) -#Current versioning information from env +# Current versioning information from env BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data + +PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null) + +# See if we have libre2-dev installed for C++ optimizations. +# In fedora and other distros, we need to tell where to find re2.pc +RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index 8e2cdf19b40..68abc68537b 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -4,9 +4,9 @@ MAKE=make GOOS=windows PREFIX=$(shell $$env:TEMP) -#Current versioning information from env -#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') -#hardcode it till i find a workaround +# Current versioning information from env +# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') +# hardcode it till I find a workaround BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1)) BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss") DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config @@ -18,3 +18,5 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 + +PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 9fc8fc86f72..a7916dc2bcf 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -35,6 +35,20 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) +type DataSourceUnavailableError struct { + Name string + Err error +} + +func (e *DataSourceUnavailableError) Error() string { + return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err) +} + +func (e *DataSourceUnavailableError) Unwrap() error { + return e.Err +} + + // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -75,6 +89,10 @@ func GetDataSourceIface(dataSourceType string) DataSource { return source() } +// DataSourceConfigure creates and returns a DataSource object from a configuration, +// if the configuration is not valid it returns an error. +// If the datasource can't be run (eg. journalctl not available), it still returns an error which +// can be checked for the appropriate action. func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { // we dump it back to []byte, because we want to decode the yaml blob twice: // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource @@ -100,7 +118,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err) + return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { @@ -173,10 +191,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) + idx := -1 for { var sub configuration.DataSourceCommonCfg - var idx int err = dec.Decode(&sub) + idx += 1 if err != nil { if !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) @@ -193,7 +212,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) - idx += 1 continue } return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) @@ -208,6 +226,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, sub.UniqueId = uniqueId src, err := DataSourceConfigure(sub) if err != nil { + var dserr *DataSourceUnavailableError + if errors.As(err, &dserr) { + log.Error(err) + continue + } return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err) } if sub.TransformExpr != "" { @@ -218,7 +241,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, transformRuntimes[uniqueId] = vm } sources = append(sources, *src) - idx += 1 } } return sources, nil @@ -295,6 +317,11 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { + // Don't wait if we have no sources, as it will hang forever + if len(sources) == 0 { + return nil + } + for i := 0; i < len(sources); i++ { subsrc := sources[i] //ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) @@ -330,11 +357,8 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } - // Don't wait if we have no sources, as it will hang forever - if len(sources) > 0 { - /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() - return err - } - return nil + + /*return only when acquisition is over (cat) or never (tail)*/ + err := AcquisTomb.Wait() + return err } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 6b6d5ce7106..548ecc04bb5 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -167,7 +167,7 @@ log_level: debug source: mock_cant_run wowo: ajsajasjas `, - ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", + ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro", }, } diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 1abf04c5b35..48bbe421753 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -201,7 +200,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro targetStream := "*" if cw.Config.StreamRegexp != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { - return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) + return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err) } targetStream = *cw.Config.StreamRegexp } else if cw.Config.StreamName != nil { @@ -345,8 +344,7 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig }, ) if err != nil { - newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName) - return newerr + return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) } cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } @@ -495,7 +493,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName) + newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) cfg.logger.Warningf("err : %s", newerr) return newerr } @@ -532,7 +530,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, u, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrapf(err, "while parsing %s", dsn) + return fmt.Errorf("while parsing %s: %w", dsn, err) } for k, v := range u { @@ -543,7 +541,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(v[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", v[0]) + return fmt.Errorf("unknown level %s: %w", v[0], err) } cw.logger.Logger.SetLevel(lvl) @@ -577,7 +575,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, //let's reuse our parser helper so that a ton of date formats are supported duration, err := time.ParseDuration(v[0]) if err != nil { - return errors.Wrapf(err, "unable to parse '%s' as duration", v[0]) + return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err) } cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration) start := time.Now().UTC().Add(-duration) @@ -674,7 +672,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName) + return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err) } cfg.logger.Tracef("after GetLogEventsPagesWithContext") case <-cw.t.Dying(): diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index b1808e446bb..92962697437 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -12,7 +12,6 @@ import ( dockerTypes "github.com/docker/docker/api/types" "github.com/docker/docker/client" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -80,7 +79,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &d.Config) if err != nil { - return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") + return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err) } if d.logger != nil { @@ -214,7 +213,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg parameters, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err) + return fmt.Errorf("while parsing parameters %s: %w", dsn, err) } for k, v := range parameters { @@ -225,7 +224,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg } lvl, err := log.ParseLevel(v[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", v[0]) + return fmt.Errorf("unknown level %s: %w", v[0], err) } d.logger.Logger.SetLevel(lvl) case "until": diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index c24b1733218..799e78dca3d 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -14,8 +14,6 @@ import ( "strings" "time" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/fsnotify/fsnotify" "github.com/nxadm/tail" "github.com/pkg/errors" @@ -24,6 +22,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { f.watcher, err = fsnotify.NewWatcher() if err != nil { - return errors.Wrapf(err, "Could not create fsnotify watcher") + return fmt.Errorf("could not create fsnotify watcher: %w", err) } f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) @@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { } files, err := filepath.Glob(pattern) if err != nil { - return errors.Wrap(err, "Glob failure") + return fmt.Errorf("glob failure: %w", err) } if len(files) == 0 { f.logger.Warnf("No matching files for pattern %s", pattern) @@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrap(err, "could not parse file args") + return fmt.Errorf("could not parse file args: %w", err) } for key, value := range params { switch key { @@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } f.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0]) + return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err) } f.config.MaxBufferSize = maxBufferSize default: @@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger f.logger.Debugf("Will try pattern %s", args[0]) files, err := filepath.Glob(args[0]) if err != nil { - return errors.Wrap(err, "Glob failure") + return fmt.Errorf("glob failure: %w", err) } if len(files) == 0 { @@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): err := f.watcher.Close() if err != nil { - return errors.Wrapf(err, "could not remove all inotify watches") + return fmt.Errorf("could not remove all inotify watches: %w", err) } return nil } @@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom fd, err := os.Open(filename) if err != nil { - return errors.Wrapf(err, "failed opening %s", filename) + return fmt.Errorf("failed opening %s: %w", filename, err) } defer fd.Close() @@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom gz, err := gzip.NewReader(fd) if err != nil { logger.Errorf("Failed to read gz file: %s", err) - return errors.Wrapf(err, "failed to read gz %s", filename) + return fmt.Errorf("failed to read gz %s: %w", filename, err) } defer gz.Close() scanner = bufio.NewScanner(gz) diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index ff55bc4134e..8f80d050fd9 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) { { name: "glob syntax error", config: `filename: "[asd-.log"`, - expectedErr: "Glob failure: syntax error in pattern", + expectedErr: "glob failure: syntax error in pattern", }, { name: "bad exclude regexp", @@ -150,7 +150,7 @@ filename: /`, config: ` mode: cat filename: "[*-.log"`, - expectedConfigErr: "Glob failure: syntax error in pattern", + expectedConfigErr: "glob failure: syntax error in pattern", logLevel: log.WarnLevel, expectedLines: 0, }, diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 7882cb7c29c..b060ac364c4 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -237,7 +236,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } j.logger.Logger.SetLevel(lvl) case "since": diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 085751cfc59..dba8daf7592 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -10,7 +10,6 @@ import ( "strconv" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/segmentio/kafka-go" log "github.com/sirupsen/logrus" @@ -93,12 +92,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error { dialer, err := k.Config.NewDialer() if err != nil { - return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) + return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err) } k.Reader, err = k.Config.NewReader(dialer) if err != nil { - return errors.Wrapf(err, "cannote create %s reader", dataSourceName) + return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err) } if k.Reader == nil { @@ -149,7 +148,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error { if err == io.EOF { return nil } - k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName)) + k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) } l := types.Line{ Raw: string(m.Value), @@ -181,7 +180,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic) if err := k.Reader.Close(); err != nil { - return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic) + return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err) } return nil } @@ -264,7 +263,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er rConf.GroupID = kc.GroupID } if err := rConf.Validate(); err != nil { - return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration") + return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err) } return kafka.NewReader(rConf), nil } diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index b37d0e7b7a7..950d33a62e2 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -8,13 +8,14 @@ import ( "testing" "time" - "github.com/crowdsecurity/go-cs-lib/pkg/cstest" - - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/segmentio/kafka-go" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" - "gotest.tools/v3/assert" + + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) func TestConfigure(t *testing.T) { @@ -178,7 +179,7 @@ topic: crowdsecplaintext`), subLogger) break READLOOP } } - assert.Equal(t, ts.expectedLines, actualLines) + require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) @@ -254,7 +255,7 @@ tls: break READLOOP } } - assert.Equal(t, ts.expectedLines, actualLines) + require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index 60cdc3751e7..cc263da4f7d 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/kinesis" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -124,7 +123,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { - return errors.Wrap(err, "Cannot parse kinesis datasource configuration") + return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err) } if k.Config.Mode == "" { @@ -218,7 +217,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea return nil default: k.logger.Errorf("Error while waiting for consumer deregistration: %s", err) - return errors.Wrap(err, "Cannot describe stream consumer") + return fmt.Errorf("cannot describe stream consumer: %w", err) } } time.Sleep(time.Millisecond * 200 * time.Duration(i+1)) @@ -236,12 +235,12 @@ func (k *KinesisSource) DeregisterConsumer() error { switch err.(type) { case *kinesis.ResourceNotFoundException: default: - return errors.Wrap(err, "Cannot deregister stream consumer") + return fmt.Errorf("cannot deregister stream consumer: %w", err) } } err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN) if err != nil { - return errors.Wrap(err, "Cannot wait for consumer deregistration") + return fmt.Errorf("cannot wait for consumer deregistration: %w", err) } return nil } @@ -253,7 +252,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error { ConsumerARN: aws.String(consumerARN), }) if err != nil { - return errors.Wrap(err, "Cannot describe stream consumer") + return fmt.Errorf("cannot describe stream consumer: %w", err) } if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" { k.logger.Debugf("Consumer %s is active", consumerARN) @@ -272,11 +271,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu StreamARN: aws.String(k.Config.StreamARN), }) if err != nil { - return nil, errors.Wrap(err, "Cannot register stream consumer") + return nil, fmt.Errorf("cannot register stream consumer: %w", err) } err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN) if err != nil { - return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active") + return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err) } return streamConsumer, nil } @@ -339,7 +338,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven logger.Infof("Subscribed shard reader is dying") err := reader.Close() if err != nil { - return errors.Wrap(err, "Cannot close kinesis subscribed shard reader") + return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err) } return nil case event, ok := <-reader.Events(): @@ -362,7 +361,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R StreamName: aws.String(arn.Resource[7:]), }) if err != nil { - return errors.Wrap(err, "Cannot list shards for enhanced_read") + return fmt.Errorf("cannot list shards for enhanced_read: %w", err) } for _, shard := range shards.Shards { @@ -373,7 +372,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R ConsumerARN: streamConsumer.Consumer.ConsumerARN, }) if err != nil { - return errors.Wrap(err, "Cannot subscribe to shard") + return fmt.Errorf("cannot subscribe to shard: %w", err) } k.shardReaderTomb.Go(func() error { return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:]) @@ -385,7 +384,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { parsedARN, err := arn.Parse(k.Config.StreamARN) if err != nil { - return errors.Wrap(err, "Cannot parse stream ARN") + return fmt.Errorf("cannot parse stream ARN: %w", err) } if !strings.HasPrefix(parsedARN.Resource, "stream/") { return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN) @@ -395,12 +394,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { k.logger.Info("starting kinesis acquisition with enhanced fan-out") err = k.DeregisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot deregister consumer") + return fmt.Errorf("cannot deregister consumer: %w", err) } streamConsumer, err := k.RegisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot register consumer") + return fmt.Errorf("cannot register consumer: %w", err) } for { @@ -408,7 +407,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { err = k.SubscribeToShards(parsedARN, streamConsumer, out) if err != nil { - return errors.Wrap(err, "Cannot subscribe to shards") + return fmt.Errorf("cannot subscribe to shards: %w", err) } select { case <-t.Dying(): @@ -417,7 +416,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { _ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves err = k.DeregisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot deregister consumer") + return fmt.Errorf("cannot deregister consumer: %w", err) } return nil case <-k.shardReaderTomb.Dying(): @@ -440,7 +439,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)}) if err != nil { logger.Errorf("Cannot get shard iterator: %s", err) - return errors.Wrap(err, "Cannot get shard iterator") + return fmt.Errorf("cannot get shard iterator: %w", err) } it := sharIt.ShardIterator //AWS recommends to wait for a second between calls to GetRecords for a given shard @@ -461,7 +460,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro continue default: logger.Error("Cannot get records") - return errors.Wrap(err, "Cannot get records") + return fmt.Errorf("cannot get records: %w", err) } } k.ParseAndPushRecords(records.Records, out, logger, shardId) @@ -486,7 +485,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error StreamName: aws.String(k.Config.StreamName), }) if err != nil { - return errors.Wrap(err, "Cannot list shards") + return fmt.Errorf("cannot list shards: %w", err) } k.shardReaderTomb = &tomb.Tomb{} for _, shard := range shards.Shards { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index f65a0aa579a..24354738114 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -8,16 +8,16 @@ import ( "net/http" "strings" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" "k8s.io/apiserver/pkg/apis/audit" + + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type KubernetesAuditConfiguration struct { @@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { k8sConfig := KubernetesAuditConfiguration{} err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig) if err != nil { - return errors.Wrap(err, "Cannot parse k8s-audit configuration") + return fmt.Errorf("cannot parse k8s-audit configuration: %w", err) } ka.config = k8sConfig @@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t t.Go(func() error { err := ka.server.ListenAndServe() if err != nil && err != http.ErrServerClosed { - return errors.Wrap(err, "k8s-audit server failed") + return fmt.Errorf("k8s-audit server failed: %w", err) } return nil }) diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 4ba31f43b58..96026f55e69 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "io" "net/url" @@ -21,13 +22,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type S3Configuration struct { @@ -563,7 +564,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrap(err, "could not parse s3 args") + return fmt.Errorf("could not parse s3 args: %w", err) } for key, value := range params { switch key { @@ -573,7 +574,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } s.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -582,7 +583,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return errors.Wrapf(err, "invalid value for 'max_buffer_size'") + return fmt.Errorf("invalid value for 'max_buffer_size': %w", err) } s.logger.Debugf("Setting max buffer size to %d", maxBufferSize) s.Config.MaxBufferSize = maxBufferSize diff --git a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go index 088ab0d95e1..7118c295b54 100644 --- a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go +++ b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -31,18 +30,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error { s.port = port udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port)) if err != nil { - return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr) + return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err) } udpConn, err := net.ListenUDP("udp", udpAddr) if err != nil { - return errors.Wrapf(err, "could not listen on port %d", s.port) + return fmt.Errorf("could not listen on port %d: %w", s.port, err) } s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port) s.udpConn = udpConn err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond)) if err != nil { - return errors.Wrap(err, "could not set read deadline on UDP socket") + return fmt.Errorf("could not set read deadline on UDP socket: %w", err) } return nil } @@ -87,7 +86,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb { func (s *SyslogServer) KillServer() error { err := s.udpConn.Close() if err != nil { - return errors.Wrap(err, "could not close UDP connection") + return fmt.Errorf("could not close UDP connection: %w", err) } close(s.channel) return nil diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 948f3d005e8..840e372007b 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -100,7 +99,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &s.config) if err != nil { - return errors.Wrap(err, "Cannot parse syslog configuration") + return fmt.Errorf("cannot parse syslog configuration: %w", err) } if s.config.Addr == "" { @@ -140,7 +139,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) s.server.SetChannel(c) err := s.server.Listen(s.config.Addr, s.config.Port) if err != nil { - return errors.Wrap(err, "could not start syslog server") + return fmt.Errorf("could not start syslog server: %w", err) } s.serverTomb = s.server.StartServer() t.Go(func() error { diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index b8d1b7fc1aa..dd2ba2975ff 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - "github.com/crowdsecurity/crowdsec/pkg/models" qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) // type ApiAlerts service @@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. u := fmt.Sprintf("%s/alerts", s.client.URLPrefix) params, err := qs.Values(opts) if err != nil { - return nil, nil, errors.Wrap(err, "building query") + return nil, nil, fmt.Errorf("building query: %w", err) } if len(params) > 0 { URI = fmt.Sprintf("%s?%s", u, params.Encode()) @@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. req, err := s.client.NewRequest(http.MethodGet, URI, nil) if err != nil { - return nil, nil, errors.Wrap(err, "building request") + return nil, nil, fmt.Errorf("building request: %w", err) } resp, err := s.client.Do(ctx, req, &alerts) if err != nil { - return nil, resp, errors.Wrap(err, "performing request") + return nil, resp, fmt.Errorf("performing request: %w", err) } return &alerts, resp, nil } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index f4ec8cabe2c..aa5039f0bc7 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -8,12 +8,13 @@ import ( "reflect" "testing" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) func TestAlertsListAsMachine(t *testing.T) { diff --git a/pkg/apiclient/auth.go b/pkg/apiclient/auth.go index 48b971a061d..84df74456b1 100644 --- a/pkg/apiclient/auth.go +++ b/pkg/apiclient/auth.go @@ -3,23 +3,21 @@ package apiclient import ( "bytes" "encoding/json" - "math/rand" - "sync" - "time" - - //"errors" "fmt" "io" + "math/rand" "net/http" "net/http/httputil" "net/url" + "sync" + "time" - "github.com/crowdsecurity/crowdsec/pkg/fflag" - "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/go-openapi/strfmt" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - //"google.golang.org/appengine/log" + + "github.com/crowdsecurity/crowdsec/pkg/fflag" + "github.com/crowdsecurity/crowdsec/pkg/models" ) type APIKeyTransport struct { @@ -169,11 +167,11 @@ func (t *JWTTransport) refreshJwtToken() error { enc.SetEscapeHTML(false) err = enc.Encode(auth) if err != nil { - return errors.Wrap(err, "could not encode jwt auth body") + return fmt.Errorf("could not encode jwt auth body: %w", err) } req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf) if err != nil { - return errors.Wrap(err, "could not create request") + return fmt.Errorf("could not create request: %w", err) } req.Header.Add("Content-Type", "application/json") client := &http.Client{ @@ -196,7 +194,7 @@ func (t *JWTTransport) refreshJwtToken() error { resp, err := client.Do(req) if err != nil { - return errors.Wrap(err, "could not get jwt token") + return fmt.Errorf("could not get jwt token: %w", err) } log.Debugf("auth-jwt : http %d", resp.StatusCode) @@ -217,10 +215,10 @@ func (t *JWTTransport) refreshJwtToken() error { } if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return errors.Wrap(err, "unable to decode response") + return fmt.Errorf("unable to decode response: %w", err) } if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil { - return errors.Wrap(err, "unable to parse jwt expiration") + return fmt.Errorf("unable to parse jwt expiration: %w", err) } t.Token = response.Token @@ -263,7 +261,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err != nil { /*we had an error (network error for example, or 401 because token is refused), reset the token ?*/ t.Token = "" - return resp, errors.Wrapf(err, "performing jwt auth") + return resp, fmt.Errorf("performing jwt auth: %w", err) } log.Debugf("resp-jwt: %d", resp.StatusCode) diff --git a/pkg/apiclient/auth_service.go b/pkg/apiclient/auth_service.go index 26ad80c0c55..64284902e8c 100644 --- a/pkg/apiclient/auth_service.go +++ b/pkg/apiclient/auth_service.go @@ -21,7 +21,6 @@ type enrollRequest struct { } func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) { - u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodDelete, u, nil) if err != nil { @@ -36,7 +35,6 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) } func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) { - u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, ®istration) diff --git a/pkg/apiclient/auth_service_test.go b/pkg/apiclient/auth_service_test.go index 6236cf0417b..32ba1890f62 100644 --- a/pkg/apiclient/auth_service_test.go +++ b/pkg/apiclient/auth_service_test.go @@ -10,11 +10,12 @@ import ( "net/url" "testing" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" ) type BasicMockPayload struct { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index a7db1ed3380..d95f7749041 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -11,7 +11,6 @@ import ( "net/url" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) var ( @@ -125,9 +124,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { /*if we have http status, return it*/ if err != nil { if resp != nil && resp.Response != nil { - return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err) + return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err) } - return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err) + return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err) } return c, nil @@ -166,7 +165,7 @@ func CheckResponse(r *http.Response) error { if err == nil && data != nil { err := json.Unmarshal(data, errorResponse) if err != nil { - return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode) + return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err) } } else { errorResponse.Message = new(string) diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index c5076904147..9e082cf51cf 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) func TestNewRequestInvalid(t *testing.T) { diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index ef52a60abda..08f56730b86 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -9,9 +9,8 @@ import ( "runtime" "testing" - "github.com/stretchr/testify/assert" - log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/crowdsecurity/go-cs-lib/pkg/version" ) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index 054c51a9ceb..e96394f5611 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -6,14 +6,15 @@ import ( "fmt" "net/http" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" "github.com/crowdsecurity/crowdsec/pkg/types" - qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) type DecisionsService service diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index 935ddcea599..ab7e46e644f 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -8,14 +8,15 @@ import ( "reflect" "testing" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestDecisionsList(t *testing.T) { diff --git a/pkg/apiclient/decisions_sync_service.go b/pkg/apiclient/decisions_sync_service.go index e5284f4d9e7..57999691f21 100644 --- a/pkg/apiclient/decisions_sync_service.go +++ b/pkg/apiclient/decisions_sync_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) type DecisionDeleteService service @@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix) req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions) if err != nil { - return nil, nil, errors.Wrap(err, "while building request") + return nil, nil, fmt.Errorf("while building request: %w", err) } resp, err := d.client.Do(ctx, req, &response) if err != nil { - return nil, resp, errors.Wrap(err, "while performing request") + return nil, resp, fmt.Errorf("while performing request: %w", err) } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Decisions delete response : http %s", resp.Response.Status) diff --git a/pkg/apiclient/signal.go b/pkg/apiclient/signal.go index 27d2e3693e6..2dceb815754 100644 --- a/pkg/apiclient/signal.go +++ b/pkg/apiclient/signal.go @@ -8,7 +8,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) type SignalService service @@ -19,12 +18,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque u := fmt.Sprintf("%s/signals", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, &signals) if err != nil { - return nil, nil, errors.Wrap(err, "while building request") + return nil, nil, fmt.Errorf("while building request: %w", err) } resp, err := s.client.Do(ctx, req, &response) if err != nil { - return nil, resp, errors.Wrap(err, "while performing request") + return nil, resp, fmt.Errorf("while performing request: %w", err) } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Signal push response : http %s", resp.Response.Status) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 3791a9a3c9b..b15cf21d6bd 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -33,7 +33,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -var ( +const ( + // delta values must be smaller than the interval pullIntervalDefault = time.Hour * 2 pullIntervalDelta = 5 * time.Minute pushIntervalDefault = time.Second * 10 @@ -71,7 +72,12 @@ type apic struct { // randomDuration returns a duration value between d-delta and d+delta func randomDuration(d time.Duration, delta time.Duration) time.Duration { - return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64())) + ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta + // ticker interval must be > 0 (nanoseconds) + if ret <= 0 { + return 1 + } + return ret } func (a *apic) FetchScenariosListFromDB() ([]string, error) { @@ -822,80 +828,6 @@ func (a *apic) Pull() error { } } -func (a *apic) GetMetrics() (*models.Metrics, error) { - metric := &models.Metrics{ - ApilVersion: ptr.Of(version.String()), - Machines: make([]*models.MetricsAgentInfo, 0), - Bouncers: make([]*models.MetricsBouncerInfo, 0), - } - machines, err := a.dbClient.ListMachines() - if err != nil { - return metric, err - } - bouncers, err := a.dbClient.ListBouncers() - if err != nil { - return metric, err - } - var lastpush string - for _, machine := range machines { - if machine.LastPush == nil { - lastpush = time.Time{}.String() - } else { - lastpush = machine.LastPush.String() - } - m := &models.MetricsAgentInfo{ - Version: machine.Version, - Name: machine.MachineId, - LastUpdate: machine.UpdatedAt.String(), - LastPush: lastpush, - } - metric.Machines = append(metric.Machines, m) - } - - for _, bouncer := range bouncers { - m := &models.MetricsBouncerInfo{ - Version: bouncer.Version, - CustomName: bouncer.Name, - Name: bouncer.Type, - LastPull: bouncer.LastPull.String(), - } - metric.Bouncers = append(metric.Bouncers, m) - } - return metric, nil -} - -func (a *apic) SendMetrics(stop chan (bool)) { - defer trace.CatchPanic("lapi/metricsToAPIC") - - ticker := time.NewTicker(a.metricsIntervalFirst) - - log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval) - - for { - metrics, err := a.GetMetrics() - if err != nil { - log.Errorf("unable to get metrics (%s), will retry", err) - } - _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) - if err != nil { - log.Errorf("capi metrics: failed: %s", err) - } else { - log.Infof("capi metrics: metrics sent successfully") - } - - select { - case <-stop: - return - case <-ticker.C: - ticker.Reset(a.metricsInterval) - case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? - a.pullTomb.Kill(nil) - a.pushTomb.Kill(nil) - return - } - } -} - func (a *apic) Shutdown() { a.pushTomb.Kill(nil) a.pullTomb.Kill(nil) diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go new file mode 100644 index 00000000000..4befcf50ccb --- /dev/null +++ b/pkg/apiserver/apic_metrics.go @@ -0,0 +1,145 @@ +package apiserver + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/exp/slices" + + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/models" +) + +func (a *apic) GetMetrics() (*models.Metrics, error) { + machines, err := a.dbClient.ListMachines() + if err != nil { + return nil, err + } + + machinesInfo := make([]*models.MetricsAgentInfo, len(machines)) + + for i, machine := range machines { + machinesInfo[i] = &models.MetricsAgentInfo{ + Version: machine.Version, + Name: machine.MachineId, + LastUpdate: machine.UpdatedAt.String(), + LastPush: ptr.OrEmpty(machine.LastPush).String(), + } + } + + bouncers, err := a.dbClient.ListBouncers() + if err != nil { + return nil, err + } + + bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers)) + + for i, bouncer := range bouncers { + bouncersInfo[i] = &models.MetricsBouncerInfo{ + Version: bouncer.Version, + CustomName: bouncer.Name, + Name: bouncer.Type, + LastPull: bouncer.LastPull.String(), + } + } + + return &models.Metrics{ + ApilVersion: ptr.Of(version.String()), + Machines: machinesInfo, + Bouncers: bouncersInfo, + }, nil +} + +func (a *apic) fetchMachineIDs() ([]string, error) { + machines, err := a.dbClient.ListMachines() + if err != nil { + return nil, err + } + + ret := make([]string, len(machines)) + for i, machine := range machines { + ret[i] = machine.MachineId + } + // sorted slices are required for the slices.Equal comparison + slices.Sort(ret) + return ret, nil +} + +// SendMetrics sends metrics to the API server until it receives a stop signal. +// +// Metrics are sent at start, then at the randomized metricsIntervalFirst, +// then at regular metricsInterval. If a change is detected in the list +// of machines, the next metrics are sent immediately. +func (a *apic) SendMetrics(stop chan (bool)) { + defer trace.CatchPanic("lapi/metricsToAPIC") + + // verify the list of machines every interval + const checkInt = 20 * time.Second + + // intervals must always be > 0 + metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval} + + log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", + metInts[1].Round(time.Second), metInts[2]) + + count := -1 + nextMetInt := func() time.Duration { + if count < len(metInts)-1 { + count++ + } + return metInts[count] + } + + // store the list of machine IDs to compare + // with the next list + machineIDs := []string{} + + reloadMachineIDs := func() { + ids, err := a.fetchMachineIDs() + if err != nil { + log.Debugf("unable to get machines (%s), will retry", err) + return + } + machineIDs = ids + } + + checkTicker := time.NewTicker(checkInt) + metTicker := time.NewTicker(nextMetInt()) + + for { + select { + case <-stop: + checkTicker.Stop() + metTicker.Stop() + return + case <-checkTicker.C: + oldIDs := machineIDs + reloadMachineIDs() + if !slices.Equal(oldIDs, machineIDs) { + log.Infof("capi metrics: machines changed, immediate send") + metTicker.Reset(1) + } + case <-metTicker.C: + metrics, err := a.GetMetrics() + if err != nil { + log.Errorf("unable to get metrics (%s), will retry", err) + } + log.Info("capi metrics: sending") + _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) + if err != nil { + log.Errorf("capi metrics: failed: %s", err) + } + metTicker.Reset(nextMetInt()) + case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? + checkTicker.Stop() + metTicker.Stop() + a.pullTomb.Kill(nil) + a.pushTomb.Kill(nil) + return + } + } +} diff --git a/pkg/apiserver/apic_metrics_test.go b/pkg/apiserver/apic_metrics_test.go new file mode 100644 index 00000000000..7e37ea1e9d2 --- /dev/null +++ b/pkg/apiserver/apic_metrics_test.go @@ -0,0 +1,101 @@ +package apiserver + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/apiclient" +) + +func TestAPICSendMetrics(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedCalls int + setUp func(*apic) + metricsInterval time.Duration + }{ + { + name: "basic", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) {}, + }, + { + name: "with some metrics", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) { + api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) + api.dbClient.Ent.Machine.Create(). + SetMachineId("1234"). + SetPassword(testPassword.String()). + SetIpAddress("1.2.3.4"). + SetScenarios("crowdsecurity/test"). + SetLastPush(time.Time{}). + SetUpdatedAt(time.Time{}). + ExecX(context.Background()) + + api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) + api.dbClient.Ent.Bouncer.Create(). + SetIPAddress("1.2.3.6"). + SetName("someBouncer"). + SetAPIKey("foobar"). + SetRevoked(false). + SetLastPull(time.Time{}). + ExecX(context.Background()) + }, + }, + } + + httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) + httpmock.Activate() + defer httpmock.Deactivate() + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + url, err := url.ParseRequestURI("http://api.crowdsec.net/") + require.NoError(t, err) + + apiClient, err := apiclient.NewDefaultClient( + url, + "/api", + fmt.Sprintf("crowdsec/%s", version.String()), + nil, + ) + require.NoError(t, err) + + api := getAPIC(t) + api.pushInterval = time.Millisecond + api.pushIntervalFirst = time.Millisecond + api.apiClient = apiClient + api.metricsInterval = tc.metricsInterval + api.metricsIntervalFirst = tc.metricsInterval + tc.setUp(api) + + stop := make(chan bool) + httpmock.ZeroCallCounters() + go api.SendMetrics(stop) + time.Sleep(tc.duration) + stop <- true + + info := httpmock.GetCallCountInfo() + noResponderCalls := info["NO_RESPONDER"] + responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] + assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) + assert.Zero(t, noResponderCalls) + }) + } +} diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 65ca299911b..8aeb092cd42 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -1057,90 +1057,6 @@ func TestAPICPush(t *testing.T) { } } -func TestAPICSendMetrics(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedCalls int - setUp func(*apic) - metricsInterval time.Duration - }{ - { - name: "basic", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) {}, - }, - { - name: "with some metrics", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) { - api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) - api.dbClient.Ent.Machine.Create(). - SetMachineId("1234"). - SetPassword(testPassword.String()). - SetIpAddress("1.2.3.4"). - SetScenarios("crowdsecurity/test"). - SetLastPush(time.Time{}). - SetUpdatedAt(time.Time{}). - ExecX(context.Background()) - - api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) - api.dbClient.Ent.Bouncer.Create(). - SetIPAddress("1.2.3.6"). - SetName("someBouncer"). - SetAPIKey("foobar"). - SetRevoked(false). - SetLastPull(time.Time{}). - ExecX(context.Background()) - }, - }, - } - - httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) - httpmock.Activate() - defer httpmock.Deactivate() - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - url, err := url.ParseRequestURI("http://api.crowdsec.net/") - require.NoError(t, err) - - apiClient, err := apiclient.NewDefaultClient( - url, - "/api", - fmt.Sprintf("crowdsec/%s", version.String()), - nil, - ) - require.NoError(t, err) - - api := getAPIC(t) - api.pushInterval = time.Millisecond - api.pushIntervalFirst = time.Millisecond - api.apiClient = apiClient - api.metricsInterval = tc.metricsInterval - api.metricsIntervalFirst = tc.metricsInterval - tc.setUp(api) - - stop := make(chan bool) - httpmock.ZeroCallCounters() - go api.SendMetrics(stop) - time.Sleep(tc.duration) - stop <- true - - info := httpmock.GetCallCountInfo() - noResponderCalls := info["NO_RESPONDER"] - responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] - assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) - assert.Zero(t, noResponderCalls) - }) - } -} - func TestAPICPull(t *testing.T) { api := getAPIC(t) tests := []struct { diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c6510057366..3facd0f6f29 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -9,9 +9,18 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "time" + "github.com/gin-gonic/gin" + "github.com/go-co-op/gocron" + "github.com/golang-jwt/jwt/v4" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" + "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -22,13 +31,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/gin-gonic/gin" - "github.com/go-co-op/gocron" - "github.com/golang-jwt/jwt/v4" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" - "gopkg.in/tomb.v2" ) var ( @@ -116,7 +118,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { logFile := "" if config.LogMedia == "file" { - logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir) + logFile = filepath.Join(config.LogDir, "crowdsec_api.log") } if log.GetLevel() < log.DebugLevel { @@ -162,15 +164,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.CompressLogs != nil { _compress = *config.CompressLogs } - /*cf. https://github.com/natefinch/lumberjack/issues/82 - let's create the file beforehand w/ the right perms */ - // check if file exists - _, err := os.Stat(logFile) - // create file if not exists, purposefully ignore errors - if os.IsNotExist(err) { - file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600) - file.Close() - } + LogOutput := &lumberjack.Logger{ Filename: logFile, MaxSize: _maxsize, //megabytes diff --git a/pkg/apiserver/controllers/v1/controller.go b/pkg/apiserver/controllers/v1/controller.go index 3820bc10fd3..60da83d7dcb 100644 --- a/pkg/apiserver/controllers/v1/controller.go +++ b/pkg/apiserver/controllers/v1/controller.go @@ -2,17 +2,15 @@ package v1 import ( "context" + "fmt" "net" - //"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers" - middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" "github.com/crowdsecurity/crowdsec/pkg/csprofiles" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) type Controller struct { @@ -48,7 +46,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) { profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg) if err != nil { - return &Controller{}, errors.Wrapf(err, "failed to compile profiles") + return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err) } v1 := &Controller{ diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index 503f4d43d15..ce1bc8eeece 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -3,7 +3,7 @@ package v1 import ( "crypto/rand" "crypto/sha512" - "encoding/hex" + "encoding/base64" "fmt" "net/http" "strings" @@ -15,9 +15,11 @@ import ( log "github.com/sirupsen/logrus" ) -var ( +const ( APIKeyHeader = "X-Api-Key" bouncerContextKey = "bouncer_info" + // max allowed by bcrypt 72 = 54 bytes in base64 + dummyAPIKeySize = 54 ) type APIKey struct { @@ -31,7 +33,7 @@ func GenerateAPIKey(n int) (string, error) { if _, err := rand.Read(bytes); err != nil { return "", err } - return hex.EncodeToString(bytes), nil + return base64.StdEncoding.EncodeToString(bytes), nil } func NewAPIKey(dbClient *database.Client) *APIKey { @@ -82,7 +84,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { if err != nil && strings.Contains(err.Error(), "bouncer not found") { //Because we have a valid cert, automatically create the bouncer in the database if it does not exist //Set a random API key, but it will never be used - apiKey, err := GenerateAPIKey(64) + apiKey, err := GenerateAPIKey(dummyAPIKeySize) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/jwt.go b/pkg/apiserver/middlewares/v1/jwt.go index 9f69f332e69..bbd33c54420 100644 --- a/pkg/apiserver/middlewares/v1/jwt.go +++ b/pkg/apiserver/middlewares/v1/jwt.go @@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { //Machine was not found, let's create it log.Printf("machine %s not found, create it", machineID) //let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli) - pwd, err := GenerateAPIKey(64) + pwd, err := GenerateAPIKey(dummyAPIKeySize) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/tls_auth.go b/pkg/apiserver/middlewares/v1/tls_auth.go index a0b837a41c6..87ca896a8f4 100644 --- a/pkg/apiserver/middlewares/v1/tls_auth.go +++ b/pkg/apiserver/middlewares/v1/tls_auth.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ocsp" ) @@ -176,9 +175,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) ( } revoked, err := ta.isRevoked(cert, issuer) if err != nil { - //Fail securely, if we can't check the revokation status, let's consider the cert invalid + //Fail securely, if we can't check the revocation status, let's consider the cert invalid //We may change this in the future based on users feedback, but this seems the most sensible thing to do - return true, errors.Wrap(err, "could not check for client certification revokation status") + return true, fmt.Errorf("could not check for client certification revocation status: %w", err) } return revoked, nil @@ -231,7 +230,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) { revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1]) if err != nil { ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err) - return false, "", errors.Wrap(err, "could not check for client certification revokation status") + return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err) } if revoked { return false, "", fmt.Errorf("client certificate is revoked") diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 6add00c05df..9d80cd95ac1 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -4,7 +4,6 @@ import ( "fmt" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -39,7 +38,7 @@ func (c *Config) LoadCommon() error { } *k, err = filepath.Abs(*k) if err != nil { - return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) + return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) } } diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 59be93ae6c5..24ff454b78d 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -3,8 +3,6 @@ package csconfig import ( "fmt" "path/filepath" - - "github.com/pkg/errors" ) type ConfigurationPaths struct { @@ -50,7 +48,7 @@ func (c *Config) LoadConfigurationPaths() error { } *k, err = filepath.Abs(*k) if err != nil { - return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) + return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) } } diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index 41725bcf27d..ec70fb459ae 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -2,13 +2,13 @@ package csconfig import ( "bytes" + "errors" "fmt" "io" "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" "gopkg.in/yaml.v2" ) @@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error { if errors.Is(err, io.EOF) { break } - return errors.Wrapf(err, "while decoding %s", c.ProfilesPath) + return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err) } c.Profiles = append(c.Profiles, &t) } diff --git a/pkg/csplugin/broker.go b/pkg/csplugin/broker.go index 6bc3b129639..b1bd54dfd53 100644 --- a/pkg/csplugin/broker.go +++ b/pkg/csplugin/broker.go @@ -2,6 +2,7 @@ package csplugin import ( "context" + "errors" "fmt" "io" "os" @@ -14,7 +15,6 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/google/uuid" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" @@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs pb.pluginProcConfig = pluginCfg pb.pluginsTypesToDispatch = make(map[string]struct{}) if err := pb.loadConfig(configPaths.NotificationDir); err != nil { - return errors.Wrap(err, "while loading plugin config") + return fmt.Errorf("while loading plugin config: %w", err) } if err := pb.loadPlugins(configPaths.PluginDir); err != nil { - return errors.Wrap(err, "while loading plugin") + return fmt.Errorf("while loading plugin: %w", err) } pb.watcher = PluginWatcher{} pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName) @@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error { data = []byte(csstring.StrictExpand(string(data), os.LookupEnv)) _, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data}) if err != nil { - return errors.Wrapf(err, "while configuring %s", pc.Name) + return fmt.Errorf("while configuring %s: %w", pc.Name, err) } log.Infof("registered plugin %s", pc.Name) pb.notificationPluginByName[pc.Name] = pluginClient @@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { parsedConfigs := make([]PluginConfig, 0) yamlFile, err := os.Open(path) if err != nil { - return parsedConfigs, errors.Wrapf(err, "while opening %s", path) + return nil, fmt.Errorf("while opening %s: %w", path, err) } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) @@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { if errors.Is(err, io.EOF) { break } - return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err) + return nil, fmt.Errorf("while decoding %s got error %s", path, err) } // if the yaml document is empty, skip if reflect.DeepEqual(pc, PluginConfig{}) { diff --git a/pkg/csplugin/broker_suite_test.go b/pkg/csplugin/broker_suite_test.go index 4c7cdd6eb42..6e3e51407ea 100644 --- a/pkg/csplugin/broker_suite_test.go +++ b/pkg/csplugin/broker_suite_test.go @@ -14,7 +14,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) - type PluginSuite struct { suite.Suite @@ -23,21 +22,19 @@ type PluginSuite struct { // full path to the built plugin binary builtBinary string - runDir string // temporary directory for each test - pluginDir string // (config_paths.plugin_dir) - notifDir string // (config_paths.notification_dir) - pluginBinary string // full path to the plugin binary (unique for each test) - pluginConfig string // full path to the notification config (unique for each test) + runDir string // temporary directory for each test + pluginDir string // (config_paths.plugin_dir) + notifDir string // (config_paths.notification_dir) + pluginBinary string // full path to the plugin binary (unique for each test) + pluginConfig string // full path to the notification config (unique for each test) pluginBroker *PluginBroker } - func TestPluginSuite(t *testing.T) { suite.Run(t, new(PluginSuite)) } - func (s *PluginSuite) SetupSuite() { var err error @@ -57,14 +54,12 @@ func (s *PluginSuite) SetupSuite() { require.NoError(t, err, "while building dummy plugin") } - func (s *PluginSuite) TearDownSuite() { t := s.T() err := os.RemoveAll(s.buildDir) require.NoError(t, err) } - func copyFile(src string, dst string) error { s, err := os.Open(src) if err != nil { @@ -99,7 +94,6 @@ func (s *PluginSuite) TearDownTest() { s.TearDownSubTest() } - func (s *PluginSuite) SetupSubTest() { var err error t := s.T() @@ -125,7 +119,7 @@ func (s *PluginSuite) SetupSubTest() { require.NoError(t, err, "while copying built binary") err = os.Chmod(s.pluginBinary, 0o744) require.NoError(t, err, "chmod 0744 %s", s.pluginBinary) - + s.pluginConfig = path.Join(s.notifDir, "dummy.yaml") err = copyFile("testdata/dummy.yaml", s.pluginConfig) require.NoError(t, err, "while copying plugin config") diff --git a/pkg/csplugin/broker_test.go b/pkg/csplugin/broker_test.go index 467fadf4569..991b89ed20c 100644 --- a/pkg/csplugin/broker_test.go +++ b/pkg/csplugin/broker_test.go @@ -22,7 +22,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" ) - func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { return func(t *testing.T) { err := os.Chmod(s.pluginBinary, perm) @@ -30,30 +29,28 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { } } -func (s *PluginSuite) readconfig() (PluginConfig) { +func (s *PluginSuite) readconfig() PluginConfig { var config PluginConfig t := s.T() orig, err := os.ReadFile(s.pluginConfig) - require.NoError(t, err,"unable to read config file %s", s.pluginConfig) + require.NoError(t, err, "unable to read config file %s", s.pluginConfig) err = yaml.Unmarshal(orig, &config) - require.NoError(t, err,"unable to unmarshal config file") - + require.NoError(t, err, "unable to unmarshal config file") + return config } - func (s *PluginSuite) writeconfig(config PluginConfig) { t := s.T() data, err := yaml.Marshal(&config) - require.NoError(t, err,"unable to marshal config file") + require.NoError(t, err, "unable to marshal config file") err = os.WriteFile(s.pluginConfig, data, 0644) - require.NoError(t, err,"unable to write config file %s", s.pluginConfig) + require.NoError(t, err, "unable to write config file %s", s.pluginConfig) } - func (s *PluginSuite) TestBrokerInit() { tests := []struct { name string @@ -62,7 +59,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "group writable binary", @@ -349,7 +346,7 @@ func (s *PluginSuite) TestBrokerRunSimple() { DefaultEmptyTicker = 50 * time.Millisecond t := s.T() - + pb, err := s.InitBroker(nil) assert.NoError(t, err) diff --git a/pkg/csplugin/broker_win_test.go b/pkg/csplugin/broker_win_test.go index 3d7498c0dd1..01262b1fd0c 100644 --- a/pkg/csplugin/broker_win_test.go +++ b/pkg/csplugin/broker_win_test.go @@ -33,7 +33,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "no plugin dir", diff --git a/pkg/csplugin/listfiles.go b/pkg/csplugin/listfiles.go index 2dea44f4fb1..c91be03b45d 100644 --- a/pkg/csplugin/listfiles.go +++ b/pkg/csplugin/listfiles.go @@ -13,10 +13,9 @@ func listFilesAtPath(path string) ([]string, error) { return nil, err } for _, file := range files { - if ! file.IsDir() { + if !file.IsDir() { filePaths = append(filePaths, filepath.Join(path, file.Name())) } } return filePaths, nil } - diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index 8bcedaa1fae..09102ef0da1 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -27,9 +27,9 @@ func TestListFilesAtPath(t *testing.T) { require.NoError(t, err) tests := []struct { - name string - path string - want []string + name string + path string + want []string expectedErr string }{ { @@ -41,8 +41,8 @@ func TestListFilesAtPath(t *testing.T) { }, }, { - name: "invalid directory", - path: "./foo/bar/", + name: "invalid directory", + path: "./foo/bar/", expectedErr: "open ./foo/bar/: " + cstest.PathNotFoundMessage, }, } diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go index 64a1e6e7184..8ab1aa923b8 100644 --- a/pkg/csplugin/notifier.go +++ b/pkg/csplugin/notifier.go @@ -4,9 +4,10 @@ import ( "context" "fmt" - "github.com/crowdsecurity/crowdsec/pkg/protobufs" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" + + "github.com/crowdsecurity/crowdsec/pkg/protobufs" ) type Notifier interface { diff --git a/pkg/csplugin/utils.go b/pkg/csplugin/utils.go index 67707c82955..5e19dee386f 100644 --- a/pkg/csplugin/utils.go +++ b/pkg/csplugin/utils.go @@ -3,6 +3,7 @@ package csplugin import ( + "errors" "fmt" "io/fs" "math" @@ -13,8 +14,6 @@ import ( "strconv" "strings" "syscall" - - "github.com/pkg/errors" ) func CheckCredential(uid int, gid int) *syscall.SysProcAttr { @@ -35,7 +34,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { } cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group) if err != nil { - return nil, errors.Wrap(err, "while getting process attributes") + return nil, fmt.Errorf("while getting process attributes: %w", err) } cmd.SysProcAttr.Credential.NoSetGroups = true } @@ -105,17 +104,17 @@ func pluginIsValid(path string) error { // check if it exists if details, err = os.Stat(path); err != nil { - return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) + return fmt.Errorf("plugin at %s does not exist: %w", path, err) } // check if it is owned by current user currentUser, err := user.Current() if err != nil { - return errors.Wrap(err, "while getting current user") + return fmt.Errorf("while getting current user: %w", err) } currentUID, err := getUID(currentUser.Username) if err != nil { - return errors.Wrap(err, "while looking up the current uid") + return fmt.Errorf("while looking up the current uid: %w", err) } stat := details.Sys().(*syscall.Stat_t) if stat.Uid != currentUID { diff --git a/pkg/csplugin/utils_windows.go b/pkg/csplugin/utils_windows.go index 874e30021ab..dfb11aff548 100644 --- a/pkg/csplugin/utils_windows.go +++ b/pkg/csplugin/utils_windows.go @@ -13,7 +13,6 @@ import ( "syscall" "unsafe" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -54,38 +53,38 @@ func CheckPerms(path string) error { systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid)) if err != nil { - return errors.Wrap(err, "while creating SYSTEM well known sid") + return fmt.Errorf("while creating SYSTEM well known sid: %w", err) } adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid)) if err != nil { - return errors.Wrap(err, "while creating built-in Administrators well known sid") + return fmt.Errorf("while creating built-in Administrators well known sid: %w", err) } currentUser, err := user.Current() if err != nil { - return errors.Wrap(err, "while getting current user") + return fmt.Errorf("while getting current user: %w", err) } currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username) if err != nil { - return errors.Wrap(err, "while looking up current user sid") + return fmt.Errorf("while looking up current user sid: %w", err) } sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return errors.Wrap(err, "while getting owner security info") + return fmt.Errorf("while getting owner security info: %w", err) } if !sd.IsValid() { - return errors.New("security descriptor is invalid") + return fmt.Errorf("security descriptor is invalid") } owner, _, err := sd.Owner() if err != nil { - return errors.Wrap(err, "while getting owner") + return fmt.Errorf("while getting owner: %w", err) } if !owner.IsValid() { - return errors.New("owner is invalid") + return fmt.Errorf("owner is invalid") } if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) { @@ -94,7 +93,7 @@ func CheckPerms(path string) error { dacl, _, err := sd.DACL() if err != nil { - return errors.Wrap(err, "while getting DACL") + return fmt.Errorf("while getting DACL: %w", err) } if dacl == nil { @@ -102,7 +101,7 @@ func CheckPerms(path string) error { } if err != nil { - return errors.Wrap(err, "while looking up current user sid") + return fmt.Errorf("while looking up current user sid: %w", err) } rs := reflect.ValueOf(dacl).Elem() @@ -124,7 +123,7 @@ func CheckPerms(path string) error { ace := &AccessAllowedAce{} ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace))) if ret == 0 { - return errors.Wrap(windows.GetLastError(), "while getting ACE") + return fmt.Errorf("while getting ACE: %w", windows.GetLastError()) } log.Debugf("ACE %d: %+v\n", i, ace) @@ -162,14 +161,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT| windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken) if err != nil { - return nil, errors.Wrapf(err, "while opening process token") + return nil, fmt.Errorf("while opening process token: %w", err) } defer procToken.Close() err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation, windows.TokenPrimary, &token) if err != nil { - return nil, errors.Wrapf(err, "while duplicating token") + return nil, fmt.Errorf("while duplicating token: %w", err) } //Remove all privileges from the token @@ -177,7 +176,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil) if err != nil { - return nil, errors.Wrapf(err, "while adjusting token privileges") + return nil, fmt.Errorf("while adjusting token privileges: %w", err) } //Run the plugin as a medium integrity level process @@ -195,7 +194,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { (*byte)(unsafe.Pointer(tml)), tml.Size()) if err != nil { token.Close() - return nil, errors.Wrapf(err, "while setting token information") + return nil, fmt.Errorf("while setting token information: %w", err) } return &windows.SysProcAttr{ @@ -209,7 +208,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { cmd := exec.Command(binaryPath) cmd.SysProcAttr, err = getProcessAtr() if err != nil { - return nil, errors.Wrap(err, "while getting process attributes") + return nil, fmt.Errorf("while getting process attributes: %w", err) } return cmd, err } @@ -229,7 +228,7 @@ func pluginIsValid(path string) error { // check if it exists if _, err = os.Stat(path); err != nil { - return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) + return fmt.Errorf("plugin at %s does not exist", path) } // check if it is owned by root diff --git a/pkg/csplugin/watcher.go b/pkg/csplugin/watcher.go index 983a53c8977..bec0302e462 100644 --- a/pkg/csplugin/watcher.go +++ b/pkg/csplugin/watcher.go @@ -4,9 +4,10 @@ import ( "sync" "time" - "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) /* diff --git a/pkg/csplugin/watcher_test.go b/pkg/csplugin/watcher_test.go index 94d8d061715..391a94810f5 100644 --- a/pkg/csplugin/watcher_test.go +++ b/pkg/csplugin/watcher_test.go @@ -7,9 +7,12 @@ import ( "testing" "time" - "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" - "gotest.tools/v3/assert" + + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) var ctx = context.Background() @@ -64,7 +67,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Microsecond) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") resetTestTomb(&testTomb, &pw) testTomb = tomb.Tomb{} pw.Start(&testTomb) @@ -72,7 +75,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Millisecond*5) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.NilError(t, err) + require.NoError(t, err) resetTestTomb(&testTomb, &pw) // This is to avoid the int complaining } @@ -96,7 +99,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Second) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") // Channel won't contain any events since threshold is not crossed. resetWatcherAlertCounter(&pw) @@ -104,7 +107,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") // Channel will contain an event since threshold is crossed. resetWatcherAlertCounter(&pw) @@ -112,6 +115,6 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.NilError(t, err) + require.NoError(t, err) resetTestTomb(&testTomb, &pw) } diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 29e6cdf36e2..70dea3d7a52 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -6,12 +6,13 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) type Runtime struct { @@ -47,10 +48,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters)) runtime.Cfg = profile if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" { - return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess) + return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess) } if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" { - return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) + return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) } for fIdx, filter := range profile.Filters { diff --git a/pkg/csprofiles/csprofiles_test.go b/pkg/csprofiles/csprofiles_test.go index 81b21e1fb42..8adf6829134 100644 --- a/pkg/csprofiles/csprofiles_test.go +++ b/pkg/csprofiles/csprofiles_test.go @@ -5,10 +5,11 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/require" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" - "gotest.tools/v3/assert" ) var ( @@ -95,7 +96,7 @@ func TestNewProfile(t *testing.T) { } profile, _ := NewProfile(profilesCfg) fmt.Printf("expected : %+v | result : %+v", test.expectedNbProfile, len(profile)) - assert.Equal(t, test.expectedNbProfile, len(profile)) + require.Len(t, profile, test.expectedNbProfile) }) } } @@ -199,7 +200,7 @@ func TestEvaluateProfile(t *testing.T) { t.Errorf("EvaluateProfile() got1 = %v, want %v", got1, tt.expectedMatchStatus) } if tt.expectedDuration != "" { - assert.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") + require.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") } }) } diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go index 7bcf814a034..e52922571ef 100644 --- a/pkg/cticlient/example/fire.go +++ b/pkg/cticlient/example/fire.go @@ -44,6 +44,9 @@ func main() { } for _, item := range items { + if item.State == "refused" { + continue + } banDuration := time.Until(item.Expiration.Time) allItems = append(allItems, []string{ item.Ip, diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go index 1d6550d006a..2ad0a6eb34e 100644 --- a/pkg/cticlient/types.go +++ b/pkg/cticlient/types.go @@ -120,7 +120,7 @@ type FireItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` - Status string `json:"status"` + State string `json:"state"` Expiration CustomTime `json:"expiration"` } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 5e9c0ddf040..610e90ee6ee 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -148,7 +148,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { finalName := "" f, err := os.Lstat(itemPath) if err != nil { - return nil, errors.Wrapf(err, "while performing lstat on %s", itemPath) + return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) } if f.Mode()&os.ModeSymlink == 0 { @@ -158,7 +158,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*resolve the symlink to hub file*/ pathInHub, err := os.Readlink(itemPath) if err != nil { - return nil, errors.Wrapf(err, "while reading symlink of %s", itemPath) + return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) } //extract author from path fname := filepath.Base(pathInHub) @@ -240,7 +240,7 @@ func GetInstalledScenariosAsString() ([]string, error) { items, err := GetInstalledScenarios() if err != nil { - return nil, errors.Wrap(err, "while fetching scenarios") + return nil, fmt.Errorf("while fetching scenarios: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -281,7 +281,7 @@ func GetInstalledParsersAsString() ([]string, error) { items, err := GetInstalledParsers() if err != nil { - return nil, errors.Wrap(err, "while fetching parsers") + return nil, fmt.Errorf("while fetching parsers: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -308,7 +308,7 @@ func GetInstalledPostOverflowsAsString() ([]string, error) { items, err := GetInstalledPostOverflows() if err != nil { - return nil, errors.Wrap(err, "while fetching post overflows") + return nil, fmt.Errorf("while fetching post overflows: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -321,8 +321,9 @@ func GetInstalledCollectionsAsString() ([]string, error) { items, err := GetInstalledCollections() if err != nil { - return nil, errors.Wrap(err, "while fetching collections") + return nil, fmt.Errorf("while fetching collections: %w", err) } + for _, it := range items { retStr = append(retStr, it.Name) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index b0e300aa785..f91b0dcedff 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -25,7 +25,7 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -73,7 +73,7 @@ func TestItemStatus(t *testing.T) { } func TestGetters(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -134,7 +134,7 @@ func TestGetters(t *testing.T) { } func TestIndexDownload(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -155,10 +155,16 @@ func getTestCfg() (cfg *csconfig.Config) { return } -func envSetup() *csconfig.Config { +func envSetup(t *testing.T) *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) cfg := getTestCfg() + + defaultTransport := http.DefaultClient.Transport + t.Cleanup(func() { + http.DefaultClient.Transport = defaultTransport + }) + //Mock the http client http.DefaultClient.Transport = newMockTransport() @@ -321,7 +327,7 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) @@ -353,7 +359,7 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) diff --git a/pkg/types/dataset.go b/pkg/cwhub/dataset.go similarity index 70% rename from pkg/types/dataset.go rename to pkg/cwhub/dataset.go index 2684342a94d..848686be69d 100644 --- a/pkg/types/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "fmt" @@ -6,24 +6,14 @@ import ( "net/http" "os" "path" - "time" log "github.com/sirupsen/logrus" -) -type DataSource struct { - SourceURL string `yaml:"source_url"` - DestPath string `yaml:"dest_file"` - Type string `yaml:"type"` - //Control cache strategy on expensive regexps - Cache *bool `yaml:"cache"` - Strategy *string `yaml:"strategy"` - Size *int `yaml:"size"` - TTL *time.Duration `yaml:"ttl"` -} + "github.com/crowdsecurity/crowdsec/pkg/types" +) type DataSet struct { - Data []*DataSource `yaml:"data,omitempty"` + Data []*types.DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -66,7 +56,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*DataSource, dataDir string) error { +func GetData(data []*types.DataSource, dataDir string) error { for _, dataS := range data { destPath := path.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/types/dataset_test.go b/pkg/cwhub/dataset_test.go similarity index 94% rename from pkg/types/dataset_test.go rename to pkg/cwhub/dataset_test.go index 956e3316f5e..106268c01b6 100644 --- a/pkg/types/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "os" @@ -9,7 +9,7 @@ import ( "github.com/jarcoal/httpmock" ) -func TestDownladFile(t *testing.T) { +func TestDownloadFile(t *testing.T) { examplePath := "./example.txt" defer os.Remove(examplePath) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 6923c6d05c3..17b5c0e0022 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -256,7 +255,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { - data := &types.DataSet{} + data := &DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { @@ -272,7 +271,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } if download || force { - err = types.GetData(data.Data, dataFolder) + err = GetData(data.Data, dataFolder) if err != nil { return errors.Wrap(err, "while getting data") } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index af1e938d79d..4133e227250 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -4,12 +4,12 @@ import ( "fmt" "path/filepath" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/enescakir/emoji" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) // pick a hub branch corresponding to the current crowdsec version. @@ -79,11 +79,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err := DownloadLatest(csConfig.Hub, item, force, true) if err != nil { - return errors.Wrapf(err, "while downloading %s", item.Name) + return fmt.Errorf("while downloading %s: %w", item.Name, err) } if err := AddItem(obtype, item); err != nil { - return errors.Wrapf(err, "while adding %s", item.Name) + return fmt.Errorf("while adding %s: %w", item.Name, err) } if downloadOnly { @@ -93,11 +93,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err = EnableItem(csConfig.Hub, item) if err != nil { - return errors.Wrapf(err, "while enabling %s", item.Name) + return fmt.Errorf("while enabling %s: %w", item.Name, err) } if err := AddItem(obtype, item); err != nil { - return errors.Wrapf(err, "while adding %s", item.Name) + return fmt.Errorf("while adding %s: %w", item.Name, err) } log.Infof("Enabled %s", item.Name) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 143967002c0..bf6e84fb39d 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -9,7 +9,7 @@ import ( //Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -55,7 +55,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledSceanarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -103,7 +103,7 @@ func getHubIdxOrFail(t *testing.T) { // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 961e9a02e95..ecb5d0a3a7c 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,16 +2,17 @@ package cwhub import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" "sort" "strings" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) /*the walk/parser_visit function can't receive extra args*/ @@ -368,7 +369,7 @@ func GetHubIdx(hub *csconfig.Hub) error { log.Debugf("loading hub idx %s", hub.HubIndexFile) bidx, err := os.ReadFile(hub.HubIndexFile) if err != nil { - return errors.Wrap(err, "unable to read index file") + return fmt.Errorf("unable to read index file: %w", err) } ret, err := LoadPkgIndex(bidx) if err != nil { diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index ad011723661..688288ceed4 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -9,6 +9,10 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -20,9 +24,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) const ( @@ -201,7 +202,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) } } decisionDuration, err := time.ParseDuration(*decisionItem.Duration) @@ -391,7 +392,7 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in if rollbackErr != nil { log.Errorf("rollback error: %s", rollbackErr) } - return 0, 0, 0, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return 0, 0, 0, errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) } } /*bulk insert some new decisions*/ @@ -564,7 +565,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } marshallMetas, err := json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } //the serialized field is too big, let's try to progressively strip it @@ -582,7 +583,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ marshallMetas, err = json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } if event.SerializedValidator(string(marshallMetas)) == nil { valid = true @@ -611,7 +612,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert events: %s", err) } } @@ -624,7 +625,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err) } } @@ -637,14 +638,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ duration, err := time.ParseDuration(*decisionItem.Duration) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) + return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } /*if the scope is IP or Range, convert the value to integers */ if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) } } @@ -667,7 +668,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(decisionBulk) == decisionBulkSize { decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) @@ -680,7 +681,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) } @@ -719,7 +720,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(bulk) == bulkSize { alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err) + return nil, errors.Wrapf(BulkError, "bulk creating alert : %s", err) } for alertIndex, a := range alerts { ret = append(ret, strconv.Itoa(a.ID)) @@ -728,7 +729,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return []string{}, fmt.Errorf("error while updating decisions: %s", err) + return nil, fmt.Errorf("error while updating decisions: %s", err) } } } @@ -744,7 +745,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) + return nil, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) } for alertIndex, a := range alerts { @@ -754,7 +755,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return []string{}, fmt.Errorf("error while updating decisions: %s", err) + return nil, fmt.Errorf("error while updating decisions: %s", err) } } } diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 4cd32d83961..98bfd45873b 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -4,9 +4,10 @@ import ( "fmt" "time" + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/bouncer" - "github.com/pkg/errors" ) func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { diff --git a/pkg/database/config.go b/pkg/database/config.go index 90a85490f4a..8c3578ad596 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -1,9 +1,10 @@ package database import ( + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/configitem" - "github.com/pkg/errors" ) func (c *Client) GetConfigItem(key string) (*string, error) { diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 61483f9fd60..5dacc23d4c3 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -2,17 +2,17 @@ package database import ( "fmt" + "strconv" "strings" "time" - "strconv" - "entgo.io/ent/dialect/sql" + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" ) type DecisionsByScenario struct { diff --git a/pkg/database/machines.go b/pkg/database/machines.go index 48243324dcc..7a010fbfbc4 100644 --- a/pkg/database/machines.go +++ b/pkg/database/machines.go @@ -5,12 +5,12 @@ import ( "time" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + "golang.org/x/crypto/bcrypt" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/machine" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - "golang.org/x/crypto/bcrypt" ) const CapiMachineID = types.CAPIOrigin diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index a874122ff15..a616588a76b 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -175,8 +175,8 @@ func UnmarshalJSON(params ...any) (any, error) { err := json.Unmarshal([]byte(jsonBlob), &out) if err != nil { log.Errorf("UnmarshalJSON : %s", err) - return "", err + return nil, err } target[key] = out - return "", nil + return nil, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 36415f746da..c1aa4251ca1 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -7,7 +7,6 @@ import ( "path/filepath" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/pkg/errors" ) type HubTest struct { @@ -105,7 +104,7 @@ func (h *HubTest) LoadAllTests() error { for _, f := range testsFolder { if f.IsDir() { if _, err := h.LoadTestItem(f.Name()); err != nil { - return errors.Wrapf(err, "while loading %s", f.Name()) + return fmt.Errorf("while loading %s: %w", f.Name(), err) } } } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index c3e842bbb2d..1ec7c5f4480 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -10,7 +10,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -23,7 +22,7 @@ type HubTestItemConfig struct { LogType string `yaml:"log_type"` Labels map[string]string `yaml:"labels"` IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []types.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } type HubIndex struct { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 3d52f37e5c3..95400b50d1a 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -12,14 +12,14 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/fatih/color" - "github.com/pkg/errors" diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type AssertFail struct { @@ -164,7 +164,7 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, errors.Wrapf(err, "while running expression %s", expression) + return output, fmt.Errorf("while running expression %s: %w", expression, err) } return output, nil } diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index d9ec4dddcb7..2e2a4e9c8be 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -11,11 +11,11 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type ScenarioAssert struct { @@ -149,7 +149,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { env := map[string]interface{}{"results": *s.TestData} if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { - return output, err + return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // log.Warningf("Failed building debugher for %s : %s", assert, err) @@ -162,7 +162,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, errors.Wrapf(err, "while running expression %s", expression) + return nil, fmt.Errorf("while running expression %s: %w", expression, err) } return output, nil } diff --git a/pkg/leakybucket/README.md b/pkg/leakybucket/README.md index 5254f33b29f..4614eddba0a 100644 --- a/pkg/leakybucket/README.md +++ b/pkg/leakybucket/README.md @@ -2,86 +2,102 @@ ## Bucket concepts -Leakybucket is used for decision making. Under certain conditions -enriched events are poured in these buckets. When these buckets are +The Leakybucket is used for decision making. Under certain conditions, +enriched events are poured into these buckets. When these buckets are full, we raise a new event. After this event is raised the bucket is destroyed. There are many types of buckets, and we welcome any new useful design of buckets. -Usually the bucket configuration generates the creation of many -buckets. They are differenciated by a field called stackkey. When two -events arrives with the same stackkey they go in the same matching +Usually, the bucket configuration generates the creation of many +buckets. They are differentiated by a field called stackkey. When two +events arrive with the same stackkey they go in the same matching bucket. The very purpose of these buckets is to detect clients that exceed a -certain rate of attemps to do something (ssh connection, http -authentication failure, etc...). Thus, the most use stackkey field is +certain rate of attempts to do something (ssh connection, http +authentication failure, etc...). Thus, the most used stackkey field is often the source_ip. ## Standard leaky buckets Default buckets have two main configuration options: + * capacity: number of events the bucket can hold. When the capacity is reached and a new event is poured, a new event is raised. We call this type of event overflow. This is an int. + * leakspeed: duration needed for an event to leak. When an event - leaks, it disappear from the bucket. + leaks, it disappears from the bucket. ## Trigger -It's a special type of bucket with a zero capacity. Thus, when an -event is poured in a trigger, it always raises an overflow. +A Trigger is a special type of bucket with a capacity of zero. Thus, when an +event is poured into a trigger, it always raises an overflow. ## Uniq -It's a bucket working as the standard leaky bucket except for one +A Uniq is a bucket working like the standard leaky bucket except for one thing: a filter returns a property for each event and only one occurrence of this property is allowed in the bucket, thus the bucket is called uniq. ## Counter -It's a special type of bucket with an infinite capacity and an -infinite leakspeed (it never overflows, neither leaks). Nevertheless, +A Counter is a special type of bucket with an infinite capacity and an +infinite leakspeed (it never overflows, nor leaks). Nevertheless, the event is raised after a fixed duration. The option is called duration. +## Bayesian + +A Bayesian is a special bucket that runs bayesian inference instead of +counting events. Each event must have its likelihoods specified in the +yaml file under `prob_given_benign` and `prob_given_evil`. The bucket +will continue evaluating events until the posterior goes above the +threshold (triggering the overflow) or the duration (specified by leakspeed) +expires. + ## Available configuration options for buckets ### Fields for standard buckets * type: mandatory field. Must be one of "leaky", "trigger", "uniq" or "counter" -* name: mandatory field, but the value is totally open. Nevertheless + +* name: mandatory field, but the value is totally open. Nevertheless, this value will tag the events raised by the bucket. -* filter: mandatory field. It's a filter that is run when the decision - to make an event match the bucket or not. The filter have to return + +* filter: mandatory field. It's a filter that is run to decide whether + an event matches the bucket or not. The filter has to return a boolean. As a filter implementation we use https://github.com/antonmedv/expr + * capacity: [mandatory for now, shouldn't be mandatory in the final version] it's the size of the bucket. When pouring in a bucket already with size events, it overflows. -* leakspeed: leakspeed is a time duration (has to be parseable by - https://golang.org/pkg/time/#ParseDuration). After each interval an + +* leakspeed: leakspeed is a time duration (it has to be parsed by + https://golang.org/pkg/time/#ParseDuration). After each interval, an event is leaked from the bucket. + * stackkey: mandatory field. This field is used to differentiate on - which bucket ongoing events will be poured. When an unknown stackkey - is seen in an event a new bucket is created. -* on_overflow: optional field, that tells the what to do when the - bucket is returning the overflow event. As of today, the possibility - are these: "ban,1h", "Reprocess", "Delete". - Reprocess is used to send the raised event back in the event pool to - be matched agains buckets + which instance of the bucket the matching events will be poured. + When an unknown stackkey is seen in an event, a new bucket is created. + +* on_overflow: optional field, that tells what to do when the + bucket is returning the overflow event. As of today, the possibilities + are "ban,1h", "Reprocess" or "Delete". + Reprocess is used to send the raised event back to the event pool to + be matched against buckets ### Fields for special buckets #### Uniq -Uniq has an extra field uniq_filter which is too use the filter -implementation from https://github.com/antonmedv/expr. The filter must -return a string. All strins returned by this filter in the same -buckets have to be different. Thus, if a string is seen twice it is -dismissed. + * uniq_filter: an expression that must comply with the syntax defined + in https://github.com/antonmedv/expr and must return a string. + All strings returned by this filter in the same buckets have to be different. + Thus if a string is seen twice, the event is dismissed. #### Trigger @@ -89,11 +105,27 @@ Capacity and leakspeed are not relevant for this kind of bucket. #### Counter -It's a special kind of bucket that raise an event and is destroyed -after a fixed duration. The configuration field used is duration and -must be parseable by https://golang.org/pkg/time/#ParseDuration. -Nevertheless, this kind of bucket is often used with an infinite -leakspeed and an infinite capacity [capacity set to -1 for now]. + * duration: the Counter will be destroyed after this interval + has elapsed since its creation. The duration must be parsed + by https://golang.org/pkg/time/#ParseDuration. + Nevertheless, this kind of bucket is often used with an infinite + leakspeed and an infinite capacity [capacity set to -1 for now]. + +#### Bayesian + + * bayesian_prior: The prior to start with + * bayesian_threshold: The threshold for the posterior to trigger the overflow. + * bayesian_conditions: List of Bayesian conditions with likelihoods + +Bayesian Conditions are built from: + * condition: The expr for this specific condition to be true + * prob_given_evil: The likelihood an IP satisfies the condition given the fact + that it is a maliscious IP + * prob_given_benign: The likelihood an IP satisfies the condition given the fact + that it is a benign IP + * guillotine: Bool to stop the condition from getting evaluated if it has + evaluated to true once. This should be used if evaluating the condition is + computationally expensive. ## Add examples here @@ -126,17 +158,17 @@ leakspeed and an infinite capacity [capacity set to -1 for now]. [This is not dry enough to have many details here, but:] -The bucket code is triggered by `InfiniBucketify` in main.go. -There's one struct called buckets which is for now a +The bucket code is triggered by runPour in pour.go, by calling the `leaky.PourItemToHolders` function. +There is one struct called buckets which is for now a `map[string]interface{}` that holds all buckets. The key of this map -is derivated from the filter configured for the bucket and its -stackkey. This looks like complicated, but in fact it allows us to use -only one structs. This is done in buckets.go. +is derived from the filter configured for the bucket and its +stackkey. This looks complicated, but it allows us to use +only one struct. This is done in buckets.go. -On top of that the implementation define only the standard leaky -bucket. A goroutine is launched for every buckets (bucket.go). This +On top of that the implementation defines only the standard leaky +bucket. A goroutine is launched for every bucket (`bucket.go`). This goroutine manages the life of the bucket. For special buckets, hooks are defined at initialization time in -manager.go. Hooks are called when relevant by the bucket gorourine -when events are poured and/or when bucket overflows. \ No newline at end of file +manager.go. Hooks are called when relevant by the bucket goroutine +when events are poured and/or when a bucket overflows. diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go new file mode 100644 index 00000000000..bd9aaed96b4 --- /dev/null +++ b/pkg/leakybucket/bayesian.go @@ -0,0 +1,163 @@ +package leakybucket + +import ( + "fmt" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +type RawBayesianCondition struct { + ConditionalFilterName string `yaml:"condition"` + ProbGivenEvil float32 `yaml:"prob_given_evil"` + ProbGivenBenign float32 `yaml:"prob_given_benign"` + Guillotine bool `yaml:"guillotine,omitempty"` +} + +type BayesianEvent struct { + rawCondition RawBayesianCondition + conditionalFilterRuntime *vm.Program + guillotineState bool +} + +type BayesianBucket struct { + bayesianEventArray []*BayesianEvent + prior float32 + threshold float32 + posterior float32 + DumbProcessor +} + +func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { + numerator := probGivenEvil * prior + denominator := numerator + ProbGivenBenign*(1-prior) + + return numerator / denominator +} + +func (c *BayesianBucket) OnBucketInit(g *BucketFactory) error { + var err error + BayesianEventArray := make([]*BayesianEvent, len(g.BayesianConditions)) + + if conditionalExprCache == nil { + conditionalExprCache = make(map[string]vm.Program) + } + conditionalExprCacheLock.Lock() + + for index, bcond := range g.BayesianConditions { + var bayesianEvent BayesianEvent + bayesianEvent.rawCondition = bcond + err = bayesianEvent.compileCondition() + if err != nil { + return err + } + BayesianEventArray[index] = &bayesianEvent + } + conditionalExprCacheLock.Unlock() + c.bayesianEventArray = BayesianEventArray + + c.prior = g.BayesianPrior + c.threshold = g.BayesianThreshold + + return err +} + +func (c *BayesianBucket) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event { + return func(msg types.Event, l *Leaky) *types.Event { + c.posterior = c.prior + l.logger.Debugf("starting bayesian evaluation with prior: %v", c.posterior) + + for _, bevent := range c.bayesianEventArray { + err := bevent.bayesianUpdate(c, msg, l) + if err != nil { + l.logger.Errorf("bayesian update failed for %s with %s", bevent.rawCondition.ConditionalFilterName, err) + } + } + + l.logger.Debugf("value of posterior after events : %v", c.posterior) + + if c.posterior > c.threshold { + l.logger.Debugf("Bayesian bucket overflow") + l.Ovflw_ts = l.Last_ts + l.Out <- l.Queue + return nil + } + + return &msg + } +} + +func (b *BayesianEvent) bayesianUpdate(c *BayesianBucket, msg types.Event, l *Leaky) error { + var condition, ok bool + + if b.conditionalFilterRuntime == nil { + l.logger.Tracef("empty conditional filter runtime for %s", b.rawCondition.ConditionalFilterName) + return nil + } + + l.logger.Tracef("guillotine value for %s : %v", b.rawCondition.ConditionalFilterName, b.getGuillotineState()) + if b.getGuillotineState() { + l.logger.Tracef("guillotine already triggered for %s", b.rawCondition.ConditionalFilterName) + l.logger.Tracef("condition true updating prior for: %s", b.rawCondition.ConditionalFilterName) + c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) + l.logger.Tracef("new value of posterior : %v", c.posterior) + return nil + } + + l.logger.Debugf("running condition expression: %s", b.rawCondition.ConditionalFilterName) + ret, err := expr.Run(b.conditionalFilterRuntime, map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}) + if err != nil { + return fmt.Errorf("unable to run conditional filter: %s", err) + } + + l.logger.Tracef("bayesian bucket expression %s returned : %v", b.rawCondition.ConditionalFilterName, ret) + if condition, ok = ret.(bool); !ok { + return fmt.Errorf("bayesian condition unexpected non-bool return: %T", ret) + } + + l.logger.Tracef("condition %T updating prior for: %s", condition, b.rawCondition.ConditionalFilterName) + if condition { + c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) + b.triggerGuillotine() + } else { + c.posterior = updateProbability(c.posterior, 1-b.rawCondition.ProbGivenEvil, 1-b.rawCondition.ProbGivenBenign) + } + l.logger.Tracef("new value of posterior: %v", c.posterior) + + return nil +} + +func (b *BayesianEvent) getGuillotineState() bool { + if b.rawCondition.Guillotine { + return b.guillotineState + } + return false +} + +func (b *BayesianEvent) triggerGuillotine() { + b.guillotineState = true +} + +func (b *BayesianEvent) compileCondition() error { + var err error + var compiledExpr *vm.Program + + if compiled, ok := conditionalExprCache[b.rawCondition.ConditionalFilterName]; ok { + b.conditionalFilterRuntime = &compiled + return nil + } + + conditionalExprCacheLock.Unlock() + //release the lock during compile same as coditional bucket + compiledExpr, err = expr.Compile(b.rawCondition.ConditionalFilterName, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})...) + if err != nil { + return fmt.Errorf("bayesian condition compile error: %w", err) + } + b.conditionalFilterRuntime = compiledExpr + conditionalExprCacheLock.Lock() + conditionalExprCache[b.rawCondition.ConditionalFilterName] = *compiledExpr + + return nil +} diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index 004d5b9d884..286c51f113c 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -191,6 +191,10 @@ func FromFactory(bucketFactory BucketFactory) *Leaky { l.conditionalOverflow = true l.Duration = l.BucketConfig.leakspeed } + + if l.BucketConfig.Type == "bayesian" { + l.Duration = l.BucketConfig.leakspeed + } return l } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 1e212f815fe..dc1f4ed51d8 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -51,6 +51,9 @@ type BucketFactory struct { Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc. OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow + BayesianPrior float32 `yaml:"bayesian_prior"` + BayesianThreshold float32 `yaml:"bayesian_threshold"` + BayesianConditions []RawBayesianCondition `yaml:"bayesian_conditions"` //conditions for the bayesian bucket ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP BucketName string `yaml:"-"` Filename string `yaml:"-"` @@ -120,6 +123,25 @@ func ValidateFactory(bucketFactory *BucketFactory) error { if bucketFactory.leakspeed == 0 { return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed) } + } else if bucketFactory.Type == "bayesian" { + if bucketFactory.BayesianConditions == nil { + return fmt.Errorf("bayesian bucket must have bayesian conditions") + } + if bucketFactory.BayesianPrior == 0 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") + } + if bucketFactory.BayesianThreshold == 0 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") + } + if bucketFactory.BayesianPrior > 1 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") + } + if bucketFactory.BayesianThreshold > 1 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") + } + if bucketFactory.Capacity != -1 { + return fmt.Errorf("bayesian bucket must have capacity -1") + } } else { return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type) } @@ -316,6 +338,8 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) case "conditional": bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) + case "bayesian": + bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) default: return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err) } @@ -355,6 +379,11 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &ConditionalOverflow{}) } + if bucketFactory.BayesianThreshold != 0 { + bucketFactory.logger.Tracef("Adding bayesian processor") + bucketFactory.processors = append(bucketFactory.processors, &BayesianBucket{}) + } + if len(bucketFactory.Data) > 0 { for _, data := range bucketFactory.Data { if data.DestPath == "" { diff --git a/pkg/leakybucket/manager_load_test.go b/pkg/leakybucket/manager_load_test.go index bb3df75cd92..513f11ff373 100644 --- a/pkg/leakybucket/manager_load_test.go +++ b/pkg/leakybucket/manager_load_test.go @@ -119,3 +119,25 @@ func TestCounterBucketsConfig(t *testing.T) { } } + +func TestBayesianBucketsConfig(t *testing.T) { + var CfgTests = []cfgTest{ + + //basic valid counter + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, true, true}, + //bad capacity + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: 1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //missing prior + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //missing threshold + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //bad prior + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 1.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //bad threshold + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 1.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + } + if err := runTest(CfgTests); err != nil { + t.Fatalf("%s", err) + } + +} diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index d6131cd26b9..45446b8f65e 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -6,15 +6,14 @@ import ( "sort" "strconv" - "github.com/crowdsecurity/crowdsec/pkg/alertcontext" - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/antonmedv/expr" "github.com/davecgh/go-spew/spew" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" ) // SourceFromEvent extracts and formats a valid models.Source object from an Event @@ -53,7 +52,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) if !ok { @@ -128,7 +127,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) @@ -145,7 +144,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e } retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) @@ -217,7 +216,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st for _, evt := range queue.Queue { srcs, err := SourceFromEvent(evt, leaky) if err != nil { - return nil, "", errors.Wrapf(err, "while extracting scope from bucket %s", leaky.Name) + return nil, "", fmt.Errorf("while extracting scope from bucket %s: %w", leaky.Name, err) } for key, src := range srcs { if source_type == types.Undefined { @@ -276,7 +275,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { //Get the sources from Leaky/Queue sources, source_scope, err := alertFormatSource(leaky, queue) if err != nil { - return runtimeAlert, errors.Wrap(err, "unable to collect sources from bucket") + return runtimeAlert, fmt.Errorf("unable to collect sources from bucket: %w", err) } runtimeAlert.Sources = sources //Include source info in format string diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml new file mode 100644 index 00000000000..8e8c26e6f8d --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml @@ -0,0 +1,21 @@ +type: bayesian +name: test/guillotine-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: evt.Meta.http_path == "/" + prob_given_evil: 0.8 + prob_given_benign: 0.2 + guillotine : true +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml new file mode 100644 index 00000000000..05e1557cf91 --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json new file mode 100644 index 00000000000..07b7b6a6ee5 --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json @@ -0,0 +1,50 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/guillotine-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml new file mode 100644 index 00000000000..1110fb7b82d --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml @@ -0,0 +1,21 @@ +type: bayesian +name: test/multiple-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: evt.Meta.http_path == "/" + prob_given_evil: 0.8 + prob_given_benign: 0.2 + guillotine : true +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml new file mode 100644 index 00000000000..05e1557cf91 --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json new file mode 100644 index 00000000000..69454a6edf9 --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json @@ -0,0 +1,64 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "1.2.3.4", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/multiple-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml new file mode 100644 index 00000000000..21a4ab07495 --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml @@ -0,0 +1,19 @@ +type: bayesian +name: test/simple-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: any(queue.Queue, {.Meta.http_path == "/"}) + prob_given_evil: 0.8 + prob_given_benign: 0.2 +- condition: any(queue.Queue, {.Meta.ssh_user == "admin"}) + prob_given_evil: 0.9 + prob_given_benign: 0.5 +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml new file mode 100644 index 00000000000..05e1557cf91 --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json new file mode 100644 index 00000000000..a5807c4b6f0 --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json @@ -0,0 +1,50 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/simple-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/metabase/api.go b/pkg/metabase/api.go index 99cbf9ec77f..7235ff7f104 100644 --- a/pkg/metabase/api.go +++ b/pkg/metabase/api.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" ) -type APIClient struct { +type MBClient struct { CTX *sling.Sling Client *http.Client } @@ -35,15 +35,15 @@ var ( } ) -func NewAPIClient(url string) (*APIClient, error) { +func NewMBClient(url string) (*MBClient, error) { httpClient := &http.Client{Timeout: 20 * time.Second} - return &APIClient{ + return &MBClient{ CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())), Client: httpClient, }, nil } -func (h *APIClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { +func (h *MBClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { var Success interface{} var Error interface{} var resp *http.Response @@ -80,6 +80,6 @@ func (h *APIClient) Do(method string, route string, body interface{}) (interface } // Set set headers as key:value -func (h *APIClient) Set(key string, value string) { +func (h *MBClient) Set(key string, value string) { h.CTX = h.CTX.Set(key, value) } diff --git a/pkg/metabase/database.go b/pkg/metabase/database.go index 0a7890faceb..273d06dee16 100644 --- a/pkg/metabase/database.go +++ b/pkg/metabase/database.go @@ -7,14 +7,13 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" ) type Database struct { DBUrl string Model *Model Config *csconfig.DatabaseCfg - Client *APIClient + Client *MBClient Details *Details // in case mysql host is 127.0.0.1 the ip address of mysql/pgsql host will be the docker gateway since metabase run in a container } @@ -41,7 +40,7 @@ type Model struct { Schedules map[string]interface{} `json:"schedules"` } -func NewDatabase(config *csconfig.DatabaseCfg, client *APIClient, remoteDBAddr string) (*Database, error) { +func NewDatabase(config *csconfig.DatabaseCfg, client *MBClient, remoteDBAddr string) (*Database, error) { var details *Details database := Database{} @@ -80,13 +79,13 @@ func (d *Database) Update() error { data, err := json.Marshal(success) if err != nil { - return errors.Wrap(err, "update sqlite db response (marshal)") + return fmt.Errorf("update sqlite db response (marshal): %w", err) } model := Model{} if err := json.Unmarshal(data, &model); err != nil { - return errors.Wrap(err, "update sqlite db response (unmarshal)") + return fmt.Errorf("update sqlite db response (unmarshal): %w", err) } model.Details = d.Details _, errormsg, err = d.Client.Do("PUT", routes[databaseEndpoint], model) diff --git a/pkg/metabase/metabase.go b/pkg/metabase/metabase.go index c12d9166f81..cdbe65ec8df 100644 --- a/pkg/metabase/metabase.go +++ b/pkg/metabase/metabase.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "context" + "errors" "fmt" "io" "net/http" @@ -15,15 +16,14 @@ import ( "github.com/docker/docker/client" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) type Metabase struct { Config *Config - Client *APIClient + Client *MBClient Container *Container Database *Database InternalDBURL string @@ -80,7 +80,7 @@ func (m *Metabase) Init(containerName string) error { return fmt.Errorf("database '%s' not supported", m.Config.Database.Type) } - m.Client, err = NewAPIClient(m.Config.ListenURL) + m.Client, err = NewMBClient(m.Config.ListenURL) if err != nil { return err } @@ -90,7 +90,7 @@ func (m *Metabase) Init(containerName string) error { } m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, metabaseImage, DBConnectionURI, m.Config.DockerGroupID) if err != nil { - return errors.Wrap(err, "container init") + return fmt.Errorf("container init: %w", err) } return nil @@ -151,36 +151,36 @@ func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort }, } if err := metabase.Init(containerName); err != nil { - return nil, errors.Wrap(err, "metabase setup init") + return nil, fmt.Errorf("metabase setup init: %w", err) } if err := metabase.DownloadDatabase(false); err != nil { - return nil, errors.Wrap(err, "metabase db download") + return nil, fmt.Errorf("metabase db download: %w", err) } if err := metabase.Container.Create(); err != nil { - return nil, errors.Wrap(err, "container create") + return nil, fmt.Errorf("container create: %w", err) } if err := metabase.Container.Start(); err != nil { - return nil, errors.Wrap(err, "container start") + return nil, fmt.Errorf("container start: %w", err) } log.Infof("waiting for metabase to be up (can take up to a minute)") if err := metabase.WaitAlive(); err != nil { - return nil, errors.Wrap(err, "wait alive") + return nil, fmt.Errorf("wait alive: %w", err) } if err := metabase.Database.Update(); err != nil { - return nil, errors.Wrap(err, "update database") + return nil, fmt.Errorf("update database: %w", err) } if err := metabase.Scan(); err != nil { - return nil, errors.Wrap(err, "db scan") + return nil, fmt.Errorf("db scan: %w", err) } if err := metabase.ResetCredentials(); err != nil { - return nil, errors.Wrap(err, "reset creds") + return nil, fmt.Errorf("reset creds: %w", err) } return metabase, nil @@ -193,7 +193,7 @@ func (m *Metabase) WaitAlive() error { if err != nil { if strings.Contains(err.Error(), "password:did not match stored password") { log.Errorf("Password mismatch error, is your dashboard already setup ? Run 'cscli dashboard remove' to reset it.") - return errors.Wrapf(err, "Password mismatch error") + return fmt.Errorf("password mismatch error: %w", err) } log.Debugf("%+v", err) } else { @@ -215,7 +215,7 @@ func (m *Metabase) Login(username string, password string) error { } if errormsg != nil { - return errors.Wrap(err, "http login") + return fmt.Errorf("http login: %s", errormsg) } resp, ok := successmsg.(map[string]interface{}) if !ok { @@ -238,7 +238,7 @@ func (m *Metabase) Scan() error { return err } if errormsg != nil { - return errors.Wrap(err, "http scan") + return fmt.Errorf("http scan: %s", errormsg) } return nil @@ -252,10 +252,10 @@ func (m *Metabase) ResetPassword(current string, newPassword string) error { } _, errormsg, err := m.Client.Do("PUT", routes[resetPasswordEndpoint], body) if err != nil { - return errors.Wrap(err, "reset username") + return fmt.Errorf("reset username: %w", err) } if errormsg != nil { - return errors.Wrap(err, "http reset password") + return fmt.Errorf("http reset password: %s", errormsg) } return nil } @@ -275,11 +275,11 @@ func (m *Metabase) ResetUsername(username string) error { _, errormsg, err := m.Client.Do("PUT", routes[userEndpoint], body) if err != nil { - return errors.Wrap(err, "reset username") + return fmt.Errorf("reset username: %w", err) } if errormsg != nil { - return errors.Wrap(err, "http reset username") + return fmt.Errorf("http reset username: %s", errormsg) } return nil diff --git a/pkg/parser/enrich.go b/pkg/parser/enrich.go index 04652407dd6..5180b9a5fb9 100644 --- a/pkg/parser/enrich.go +++ b/pkg/parser/enrich.go @@ -1,11 +1,12 @@ package parser import ( - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) -/* should be part of a packaged shared with enrich/geoip.go */ +/* should be part of a package shared with enrich/geoip.go */ type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type InitFunc func(map[string]string) (interface{}, error) diff --git a/pkg/parser/enrich_date.go b/pkg/parser/enrich_date.go index a1dd994be97..20828af9037 100644 --- a/pkg/parser/enrich_date.go +++ b/pkg/parser/enrich_date.go @@ -3,9 +3,10 @@ package parser import ( "time" + log "github.com/sirupsen/logrus" + expr "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" ) func parseDateWithFormat(date, format string) (string, time.Time) { diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go index 5bbc0e94df6..f622e6c359a 100644 --- a/pkg/parser/enrich_dns.go +++ b/pkg/parser/enrich_dns.go @@ -3,9 +3,9 @@ package parser import ( "net" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" - //"github.com/crowdsecurity/crowdsec/pkg/parser" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) /* All plugins must export a list of function pointers for exported symbols */ diff --git a/pkg/parser/enrich_geoip.go b/pkg/parser/enrich_geoip.go index 8c25bef4464..0a263c82793 100644 --- a/pkg/parser/enrich_geoip.go +++ b/pkg/parser/enrich_geoip.go @@ -5,11 +5,11 @@ import ( "net" "strconv" - "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" - "github.com/oschwald/geoip2-golang" "github.com/oschwald/maxminddb-golang" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/enrich_unmarshal.go b/pkg/parser/enrich_unmarshal.go index 404340401f6..dce9c75d466 100644 --- a/pkg/parser/enrich_unmarshal.go +++ b/pkg/parser/enrich_unmarshal.go @@ -3,8 +3,9 @@ package parser import ( "encoding/json" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/types/grok_pattern.go b/pkg/parser/grok_pattern.go similarity index 99% rename from pkg/types/grok_pattern.go rename to pkg/parser/grok_pattern.go index bb51d2d9be0..5b3204a4201 100644 --- a/pkg/types/grok_pattern.go +++ b/pkg/parser/grok_pattern.go @@ -1,9 +1,10 @@ -package types +package parser import ( "time" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/grokky" ) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 2370d37960c..036a244cca8 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -1,24 +1,24 @@ package parser import ( + "errors" "fmt" "net" "strings" "time" "github.com/antonmedv/expr" - "github.com/crowdsecurity/grokky" - "github.com/pkg/errors" + "github.com/antonmedv/expr/vm" + "github.com/davecgh/go-spew/spew" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" - "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/grokky" + "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) type Node struct { @@ -56,11 +56,11 @@ type Node struct { SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"` //Holds a grok pattern - Grok types.GrokPattern `yaml:"grok,omitempty"` + Grok GrokPattern `yaml:"grok,omitempty"` //Statics can be present in any type of node and is executed last - Statics []types.ExtraField `yaml:"statics,omitempty"` + Statics []ExtraField `yaml:"statics,omitempty"` //Stash allows to capture data from the log line and store it in an accessible cache - Stash []types.DataCapture `yaml:"stash,omitempty"` + Stash []DataCapture `yaml:"stash,omitempty"` //Whitelists Whitelist Whitelist `yaml:"whitelist,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"` @@ -360,29 +360,27 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } //Iterate on leafs - if len(n.LeavesNodes) > 0 { - for _, leaf := range n.LeavesNodes { - ret, err := leaf.process(p, ctx, cachedExprEnv) - if err != nil { - clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) - clog.Debugf("Event leaving node : ko") - return false, err - } - clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) - if ret { - NodeState = true - /* if child is successful, stop processing */ - if n.OnSuccess == "next_stage" { - clog.Debugf("child is success, OnSuccess=next_stage, skip") - break - } - } else if !NodeHasOKGrok { - /* - If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. - If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. - */ - NodeState = false - } + for _, leaf := range n.LeavesNodes { + ret, err := leaf.process(p, ctx, cachedExprEnv) + if err != nil { + clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) + clog.Debugf("Event leaving node : ko") + return false, err + } + clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) + if ret { + NodeState = true + /* if child is successful, stop processing */ + if n.OnSuccess == "next_stage" { + clog.Debugf("child is success, OnSuccess=next_stage, skip") + break + } + } else if !NodeHasOKGrok { + /* + If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. + If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. + */ + NodeState = false } } /*todo : check if a node made the state change ?*/ @@ -402,10 +400,11 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri if n.Name != "" { NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() } + /* - Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node. + This is to apply statics when the node *has* whitelists that successfully matched the node. */ - if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist { + if len(n.Statics) > 0 && (isWhitelisted || !hasWhitelist) { clog.Debugf("+ Processing %d statics", len(n.Statics)) // if all else is good in whitelist, process node's statics err := n.ProcessStatics(n.Statics, p) @@ -453,8 +452,8 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* if the node has debugging enabled, create a specific logger with debug that will be used only for processing this node ;) */ if n.Debug { - var clog = logrus.New() - if err := types.ConfigureLogger(clog); err != nil { + var clog = log.New() + if err = types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) @@ -493,7 +492,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* handle pattern_syntax and groks */ for _, pattern := range n.SubGroks { n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value) - if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { + if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { if errors.Is(err, grokky.ErrAlreadyExist) { n.Logger.Warningf("grok '%s' already registred", pattern.Key) continue @@ -536,20 +535,18 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling grok's expression") + return fmt.Errorf("while compiling grok's expression: %w", err) } } /* load grok statics */ - if len(n.Grok.Statics) > 0 { - //compile expr statics if present - for idx := range n.Grok.Statics { - if n.Grok.Statics[idx].ExpValue != "" { - n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, - exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) - if err != nil { - return err - } + //compile expr statics if present + for idx := range n.Grok.Statics { + if n.Grok.Statics[idx].ExpValue != "" { + n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, + exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + if err != nil { + return err } } valid = true @@ -560,54 +557,53 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Stash[i].ValueExpression, err = expr.Compile(stash.Value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling stash value expression") + return fmt.Errorf("while compiling stash value expression: %w", err) } n.Stash[i].KeyExpression, err = expr.Compile(stash.Key, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling stash key expression") + return fmt.Errorf("while compiling stash key expression: %w", err) } n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL) if err != nil { - return errors.Wrap(err, "while parsing stash ttl") + return fmt.Errorf("while parsing stash ttl: %w", err) } logLvl := n.Logger.Logger.GetLevel() //init the cache, does it make sense to create it here just to be sure everything is fine ? - if err := cache.CacheInit(cache.CacheCfg{ + if err = cache.CacheInit(cache.CacheCfg{ Size: n.Stash[i].MaxMapSize, TTL: n.Stash[i].TTLVal, Name: n.Stash[i].Name, Strategy: n.Stash[i].Strategy, LogLevel: &logLvl, }); err != nil { - return errors.Wrap(err, "while initializing cache") + return fmt.Errorf("while initializing cache: %w", err) } } /* compile leafs if present */ - if len(n.LeavesNodes) > 0 { - for idx := range n.LeavesNodes { - if n.LeavesNodes[idx].Name == "" { - n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) - } - /*propagate debug/stats to child nodes*/ - if !n.LeavesNodes[idx].Debug && n.Debug { - n.LeavesNodes[idx].Debug = true - } - if !n.LeavesNodes[idx].Profiling && n.Profiling { - n.LeavesNodes[idx].Profiling = true - } - n.LeavesNodes[idx].Stage = n.Stage - err = n.LeavesNodes[idx].compile(pctx, ectx) - if err != nil { - return err - } + for idx := range n.LeavesNodes { + if n.LeavesNodes[idx].Name == "" { + n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } + /*propagate debug/stats to child nodes*/ + if !n.LeavesNodes[idx].Debug && n.Debug { + n.LeavesNodes[idx].Debug = true + } + if !n.LeavesNodes[idx].Profiling && n.Profiling { + n.LeavesNodes[idx].Profiling = true + } + n.LeavesNodes[idx].Stage = n.Stage + err = n.LeavesNodes[idx].compile(pctx, ectx) + if err != nil { + return err } valid = true } + /* load statics if present */ for idx := range n.Statics { if n.Statics[idx].ExpValue != "" { @@ -626,6 +622,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding ip %s to whitelists", net.ParseIP(v)) valid = true } + for _, v := range n.Whitelist.Cidrs { _, tnet, err := net.ParseCIDR(v) if err != nil { @@ -635,6 +632,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding cidr %s to whitelists", tnet) valid = true } + for _, filter := range n.Whitelist.Exprs { expression := &ExprWhitelist{} expression.Filter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) @@ -660,5 +658,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { if err := n.validate(pctx, ectx); err != nil { return err } + return nil } diff --git a/pkg/parser/node_test.go b/pkg/parser/node_test.go index f529cb4d40c..d85aa82a8ae 100644 --- a/pkg/parser/node_test.go +++ b/pkg/parser/node_test.go @@ -3,7 +3,6 @@ package parser import ( "testing" - "github.com/crowdsecurity/crowdsec/pkg/types" yaml "gopkg.in/yaml.v2" ) @@ -20,7 +19,7 @@ func TestParserConfigs(t *testing.T) { Valid bool }{ //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //bad filter {&Node{Debug: true, Stage: "s00", Filter: "ratata"}, false, false}, //empty node @@ -28,25 +27,25 @@ func TestParserConfigs(t *testing.T) { //bad subgrok {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true}, //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, //bad node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, //ok node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //valid node with grok sub-pattern used by name - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, //node with unexisting grok pattern - {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, + {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, //node with grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROK"), Value: string("[a-z]")}, {Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")}, - }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, + }, Grok: GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, //node with broken grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")}, {Key: string("MYGROKBIS"), Value: string("[a-z]")}, - }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, + }, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, } for idx := range CfgTests { err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{}) @@ -64,6 +63,5 @@ func TestParserConfigs(t *testing.T) { if CfgTests[idx].Valid == false && err == nil { t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests)) } - } } diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go index 089b95835b5..04d08cc2785 100644 --- a/pkg/parser/parsing_test.go +++ b/pkg/parser/parsing_test.go @@ -11,11 +11,12 @@ import ( "strings" "testing" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type TestFile struct { @@ -54,7 +55,6 @@ func TestParser(t *testing.T) { } } } - } func BenchmarkParser(t *testing.B) { @@ -90,7 +90,6 @@ func BenchmarkParser(t *testing.B) { } func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error { - var ( err error pnodes []Node @@ -112,7 +111,7 @@ func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing if err != nil { panic(err) } - if err := yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { + if err = yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { return fmt.Errorf("failed unmarshaling %s : %s", parser_cfg_file, err) } @@ -399,7 +398,7 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } for _, k := range p { - if _, err := f.WriteString(fmt.Sprintf("## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value)); err != nil { + if _, err := fmt.Fprintf(f, "## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value); err != nil { t.Fatal("failed to write to file") } fmt.Printf("%v\t%v\n", k.Key, k.Value) @@ -414,5 +413,4 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } f.Close() - } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 3c7d382d577..e6ee07ea765 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -9,24 +9,21 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" "sync" "time" - "github.com/crowdsecurity/crowdsec/pkg/types" - - "strconv" - + "github.com/antonmedv/expr" "github.com/mohae/deepcopy" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/types" ) /* ok, this is kinda experimental, I don't know how bad of an idea it is .. */ func SetTargetByName(target string, value string, evt *types.Event) bool { - if evt == nil { return false } @@ -71,8 +68,6 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { tmp = reflect.Indirect(tmp) } iter = tmp - //nolint: gosimple - break case reflect.Ptr: tmp := iter.Elem() iter = reflect.Indirect(tmp.FieldByName(f)) @@ -94,24 +89,24 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { return true } -func printStaticTarget(static types.ExtraField) string { - - if static.Method != "" { +func printStaticTarget(static ExtraField) string { + switch { + case static.Method != "": return static.Method - } else if static.Parsed != "" { + case static.Parsed != "": return fmt.Sprintf(".Parsed[%s]", static.Parsed) - } else if static.Meta != "" { + case static.Meta != "": return fmt.Sprintf(".Meta[%s]", static.Meta) - } else if static.Enriched != "" { + case static.Enriched != "": return fmt.Sprintf(".Enriched[%s]", static.Enriched) - } else if static.TargetByName != "" { + case static.TargetByName != "": return static.TargetByName - } else { + default: return "?" } } -func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) error { +func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { //we have a few cases : //(meta||key) + (static||reference||expr) var value string @@ -197,7 +192,6 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er } else { clog.Fatal("unable to process static : unknown target") } - } return nil } @@ -357,10 +351,8 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) event.Process = false return event, nil } - } event.Process = true return event, nil - } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 3bdaed1f0e9..37d43fbfe96 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -109,17 +109,16 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( continue } - if len(node.Data) > 0 { - for _, data := range node.Data { - err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) - if err != nil { - log.Error(err) - } - if data.Type == "regexp" { //cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) - } + for _, data := range node.Data { + err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) + if err != nil { + log.Error(err) + } + if data.Type == "regexp" { //cache only makes sense for regexp + exprhelpers.RegexpCacheInit(data.DestPath, *data) } } + nodes = append(nodes, node) nodesCount++ } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index b0f38f81e6c..d5d91f9320d 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -7,12 +7,13 @@ import ( "sort" "strings" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/grokky" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/fflag" - - "github.com/crowdsecurity/grokky" - log "github.com/sirupsen/logrus" ) type UnixParserCtx struct { @@ -117,7 +118,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) if err != nil { - return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err) + return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) } /* @@ -135,8 +136,8 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { log.Infof("Loading postoverflow parsers") parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx) } else { - parsers.Povfwnodes = []Node{} log.Infof("No postoverflow parsers to load") + parsers.Povfwnodes = []Node{} } if err != nil { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index c56ad40a728..e2f179fb3a1 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -4,6 +4,7 @@ import ( "net" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index 957f70a258d..b345c0d6f63 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -7,15 +7,12 @@ import ( "os/exec" "sort" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/antonmedv/expr" "github.com/blackfireio/osinfo" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - // goccyyaml "github.com/goccy/go-yaml" - - // "github.com/k0kubun/pp" "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" @@ -52,7 +49,6 @@ func validateDataSource(opaqueDS DataSourceItem) error { return nil } - // formally validate YAML commonDS := configuration.DataSourceCommonCfg{} @@ -72,7 +68,6 @@ func validateDataSource(opaqueDS DataSourceItem) error { return fmt.Errorf("source is empty") } - // source must be known ds := acquisition.GetDataSourceIface(commonDS.Source) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 4f6ef0c3375..adb5f1d436c 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -10,6 +10,7 @@ import ( "github.com/lithammer/dedent" "github.com/stretchr/testify/require" + "github.com/crowdsecurity/go-cs-lib/pkg/csstring" "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -117,16 +118,16 @@ func TestVersionCheck(t *testing.T) { {"1", ">1", false, ""}, {"1", ">=1", true, ""}, {"1.0", "<1.0", false, ""}, - {"1", "<1", true, ""}, // XXX why? - {"1.3.5", "1.3", false, ""}, // XXX ok? + {"1", "<1", false, ""}, + {"1.3.5", "1.3", true, ""}, {"1.0", "<1.0", false, ""}, {"1.0", "<=1.0", true, ""}, {"2", ">1, <3", true, ""}, {"2", "<=2, >=2.2", false, ""}, {"2.3", "~2", true, ""}, {"2.3", "=2", true, ""}, - {"1.1.1", "=1.1", false, ""}, - {"1.1.1", "1.1", false, ""}, + {"1.1.1", "=1.1", true, ""}, + {"1.1.1", "1.1", true, ""}, {"1.1", "!=1.1.1", true, ""}, {"1.1", "~1.1.1", false, ""}, {"1.1.1", "~1.1", true, ""}, @@ -136,7 +137,7 @@ func TestVersionCheck(t *testing.T) { {"19.04", "=19.4", true, ""}, {"19.04", "~19.4", true, ""}, {"1.2.3", "~1.2", true, ""}, - {"1.2.3", "!=1.2", true, ""}, + {"1.2.3", "!=1.2", false, ""}, {"1.2.3", "1.1.1 - 1.3.4", true, ""}, {"1.3.5", "1.1.1 - 1.3.4", false, ""}, {"1.3.5", "=1", true, ""}, @@ -1007,7 +1008,7 @@ func TestDetectDatasourceValidation(t *testing.T) { "DetectYaml": detectYaml, } - expectedErr, err := cstest.Interpolate(tc.expectedErr, data) + expectedErr, err := csstring.Interpolate(tc.expectedErr, data) require.NoError(err) detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) diff --git a/pkg/types/datasource.go b/pkg/types/datasource.go new file mode 100644 index 00000000000..39b83fbaf3f --- /dev/null +++ b/pkg/types/datasource.go @@ -0,0 +1,16 @@ +package types + +import ( + "time" +) + +type DataSource struct { + SourceURL string `yaml:"source_url"` + DestPath string `yaml:"dest_file"` + Type string `yaml:"type"` + //Control cache strategy on expensive regexps + Cache *bool `yaml:"cache"` + Strategy *string `yaml:"strategy"` + Size *int `yaml:"size"` + TTL *time.Duration `yaml:"ttl"` +} diff --git a/pkg/types/ip.go b/pkg/types/ip.go index 647fb4a6303..5e4d7734f2d 100644 --- a/pkg/types/ip.go +++ b/pkg/types/ip.go @@ -6,13 +6,14 @@ import ( "math" "net" "strings" - - "github.com/pkg/errors" ) +// LastAddress returns the last address of a network func LastAddress(n net.IPNet) net.IP { + // get the last address by ORing the hostmask and the IP ip := n.IP.To4() if ip == nil { + // IPv6 ip = n.IP return net.IP{ ip[0] | ^n.Mask[0], ip[1] | ^n.Mask[1], ip[2] | ^n.Mask[2], @@ -35,7 +36,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { if strings.Contains(anyIP, "/") { _, net, err := net.ParseCIDR(anyIP) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing range %s", anyIP) + return -1, 0, 0, 0, 0, fmt.Errorf("while parsing range %s: %w", anyIP, err) } return Range2Ints(*net) } @@ -47,7 +48,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { sz, start, end, err := IP2Ints(ip) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing ip %s", anyIP) + return -1, 0, 0, 0, 0, fmt.Errorf("while parsing ip %s: %w", anyIP, err) } return sz, start, end, start, end, nil @@ -58,12 +59,12 @@ func Range2Ints(network net.IPNet) (int, int64, int64, int64, int64, error) { szStart, nwStart, sfxStart, err := IP2Ints(network.IP) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrap(err, "converting first ip in range") + return -1, 0, 0, 0, 0, fmt.Errorf("converting first ip in range: %w", err) } lastAddr := LastAddress(network) szEnd, nwEnd, sfxEnd, err := IP2Ints(lastAddr) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrap(err, "transforming last address of range") + return -1, 0, 0, 0, 0, fmt.Errorf("transforming last address of range: %w", err) } if szEnd != szStart { return -1, 0, 0, 0, 0, fmt.Errorf("inconsistent size for range first(%d) and last(%d) ip", szStart, szEnd) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index caed6961eee..0485db59eaf 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,13 +2,9 @@ package types import ( "bufio" - "bytes" - "encoding/gob" "fmt" - "io" "os" "path/filepath" - "regexp" "strconv" "strings" "time" @@ -40,19 +36,9 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level if compress != nil { _compress = *compress } - /*cf. https://github.com/natefinch/lumberjack/issues/82 - let's create the file beforehand w/ the right perms */ - fname := cfgFolder + "/crowdsec.log" - // check if file exists - _, err := os.Stat(fname) - // create file if not exists, purposefully ignore errors - if os.IsNotExist(err) { - file, _ := os.OpenFile(fname, os.O_RDWR|os.O_CREATE, 0600) - file.Close() - } LogOutput = &lumberjack.Logger{ - Filename: fname, + Filename: filepath.Join(cfgFolder, "crowdsec.log"), MaxSize: _maxsize, MaxBackups: _maxfiles, MaxAge: _maxage, @@ -82,19 +68,6 @@ func ConfigureLogger(clog *log.Logger) error { return nil } -func Clone(a, b interface{}) error { - buff := new(bytes.Buffer) - enc := gob.NewEncoder(buff) - dec := gob.NewDecoder(buff) - if err := enc.Encode(a); err != nil { - return fmt.Errorf("failed cloning %T", a) - } - if err := dec.Decode(b); err != nil { - return fmt.Errorf("failed cloning %T", b) - } - return nil -} - func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") { @@ -115,67 +88,6 @@ func ParseDuration(d string) (time.Duration, error) { return duration, nil } -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return - } - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - if os.SameFile(sourceFileStat, destinationFileStat) { - return - } - } - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - return -} - func UtcNow() time.Time { return time.Now().UTC() } @@ -193,11 +105,3 @@ func GetLineCountForFile(filepath string) int { } return lc } - -// from https://github.com/acarl005/stripansi -var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") - -func StripAnsiString(str string) string { - // the byte version doesn't strip correctly - return reStripAnsi.ReplaceAllString(str, "") -} diff --git a/plugins/notifications/dummy/Makefile b/plugins/notifications/dummy/Makefile index 612ec6c864c..d45d6f19844 100644 --- a/plugins/notifications/dummy/Makefile +++ b/plugins/notifications/dummy/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/Makefile b/plugins/notifications/email/Makefile index a386625acae..ae548af0a38 100644 --- a/plugins/notifications/email/Makefile +++ b/plugins/notifications/email/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index ba260c5571d..48da2b0fae5 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -13,11 +13,11 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect golang.org/x/net v0.11.0 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index 53a3f1ac87e..82c239b6e94 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -117,8 +116,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -127,6 +124,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/plugins/notifications/http/Makefile b/plugins/notifications/http/Makefile index 44ee8c58fd5..56f49077262 100644 --- a/plugins/notifications/http/Makefile +++ b/plugins/notifications/http/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index 614d5654769..902d5f699c9 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -12,11 +12,11 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect golang.org/x/net v0.11.0 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index 79381ac19cc..13cb2946108 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -115,8 +114,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -125,6 +122,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/plugins/notifications/slack/Makefile b/plugins/notifications/slack/Makefile index e950eba92d3..f43303eb882 100644 --- a/plugins/notifications/slack/Makefile +++ b/plugins/notifications/slack/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index 8a99949d595..5373c8de6af 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -13,12 +13,12 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index f47d069925c..fc3f63a1f95 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -66,14 +66,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -124,8 +123,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -134,6 +131,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/plugins/notifications/splunk/Makefile b/plugins/notifications/splunk/Makefile index a49c87bd647..a7f04f4d0fe 100644 --- a/plugins/notifications/splunk/Makefile +++ b/plugins/notifications/splunk/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index 99de7c6b5da..e848c8d848c 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -12,11 +12,11 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect golang.org/x/net v0.11.0 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index 79381ac19cc..5148b992096 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -115,16 +114,13 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index 246c7b181de..a57492eea81 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -12,8 +12,6 @@ Patch0: crowdsec.unit.patch Patch1: user.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: git -BuildRequires: make BuildRequires: systemd Requires: crontabs %{?fc33:BuildRequires: systemd-rpm-macros} @@ -27,7 +25,6 @@ Requires: crontabs %define version_number %(echo $VERSION) %define releasever %(echo $RELEASEVER) -%global local_version v%{version_number}-%{releasever}-rpm %global name crowdsec %global __mangle_shebangs_exclude_from /usr/bin/env @@ -38,7 +35,6 @@ Requires: crontabs %patch1 %build -BUILD_VERSION=%{local_version} make build sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config/config.yaml %install diff --git a/test/README.md b/test/README.md index bbc40a38831..7f34bd3dbef 100644 --- a/test/README.md +++ b/test/README.md @@ -56,9 +56,6 @@ architectures. ## pre-requisites - `git submodule init; git submodule update` - - `go install github.com/cloudflare/cfssl/cmd/cfssl@latest` - - `go install github.com/cloudflare/cfssl/cmd/cfssljson@latest` - - `go install github.com/mikefarah/yq/v4@latest` - `base64` - `bash>=4.4` - `curl` @@ -89,7 +86,7 @@ In BATS, you write tests in the form of Bash functions that have unique descriptions (the name of the test). You can do most things that you can normally do in a shell function. If there is any error condition, the test fails. A set of functions is provided to implement assertions, and a mechanism -of `setup`/`teardown` is provided a the level of individual tests (functions) +of `setup`/`teardown` is provided at the level of individual tests (functions) or group of tests (files). The stdout/stderr of the commands within the test function are captured by @@ -129,11 +126,6 @@ included in a larger test suite. The TAP specification is pretty minimalist and some glue may be needed. -Other tools that you can find useful: - - - [mikefarah/yq](https://github.com/mikefarah/yq) - to parse and update YAML files on the fly - - [aliou/bats.vim](https://github.com/aliou/bats.vim) - for syntax highlighting (use bash otherwise) - # setup and teardown If you have read the bats-core tutorial linked above, you are aware of the diff --git a/test/ansible/debug_tools.yml b/test/ansible/debug_tools.yml index 15baa7cab33..769a973fe95 100644 --- a/test/ansible/debug_tools.yml +++ b/test/ansible/debug_tools.yml @@ -14,3 +14,5 @@ - zsh-autosuggestions - zsh-syntax-highlighting - zsh-theme-powerlevel9k + when: + - ansible_facts.os_family == "Debian" diff --git a/test/ansible/provision_dependencies.yml b/test/ansible/provision_dependencies.yml index 891bcc16e11..bcfe8fccafb 100644 --- a/test/ansible/provision_dependencies.yml +++ b/test/ansible/provision_dependencies.yml @@ -13,6 +13,8 @@ - crowdsecurity.testing.git - crowdsecurity.testing.gcc - crowdsecurity.testing.gnu_make + - crowdsecurity.testing.pkg_config + - crowdsecurity.testing.re2 - crowdsecurity.testing.bats_requirements - name: "Install Postgres" diff --git a/test/ansible/requirements.yml b/test/ansible/requirements.yml index b1a28b70a02..70f8ca394db 100644 --- a/test/ansible/requirements.yml +++ b/test/ansible/requirements.yml @@ -11,11 +11,11 @@ roles: - src: gantsign.golang collections: + - name: ansible.posix - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git type: git - version: main + version: v0.0.4 # - name: crowdsecurity.testing # source: ../../../crowdsecurity.testing # type: dir - diff --git a/test/ansible/roles/make_fixture/tasks/main.yml b/test/ansible/roles/make_fixture/tasks/main.yml index 39f3e1785b9..305cec3a697 100644 --- a/test/ansible/roles/make_fixture/tasks/main.yml +++ b/test/ansible/roles/make_fixture/tasks/main.yml @@ -1,5 +1,6 @@ # vim: set ft=yaml.ansible: --- + - name: "Set make_cmd = make (!bsd)" ansible.builtin.set_fact: make_cmd: make @@ -17,7 +18,8 @@ block: - name: "Make bats-build bats-fixture" ansible.builtin.command: - cmd: "{{ make_cmd }} bats-build bats-fixture" + # static build and we don't have to mess with LD_LIBRARY_PATH + cmd: "{{ make_cmd }} bats-build bats-fixture BUILD_STATIC=1" chdir: "{{ ansible_env.HOME }}/crowdsec" creates: "{{ ansible_env.HOME }}/crowdsec/test/local-init/init-config-data.tar" environment: diff --git a/test/ansible/vagrant/alma-8/Vagrantfile b/test/ansible/vagrant/alma-8/Vagrantfile index 4b42adb3a06..e246c77ff48 100644 --- a/test/ansible/vagrant/alma-8/Vagrantfile +++ b/test/ansible/vagrant/alma-8/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + sudo dnf config-manager --set-enabled powertools + SHELL end common = '../common' diff --git a/test/ansible/vagrant/alma-9/Vagrantfile b/test/ansible/vagrant/alma-9/Vagrantfile index 0ac3e5fe3d5..9c3d1b67ccf 100644 --- a/test/ansible/vagrant/alma-9/Vagrantfile +++ b/test/ansible/vagrant/alma-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install kitty-terminfo + sudo dnf config-manager --set-enabled crb + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-7/Vagrantfile b/test/ansible/vagrant/centos-7/Vagrantfile index d7ac021d21d..fd207359f5b 100644 --- a/test/ansible/vagrant/centos-7/Vagrantfile +++ b/test/ansible/vagrant/centos-7/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'centos/7' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-8/Vagrantfile b/test/ansible/vagrant/centos-8/Vagrantfile index 24c37ada97d..a88d621c442 100644 --- a/test/ansible/vagrant/centos-8/Vagrantfile +++ b/test/ansible/vagrant/centos-8/Vagrantfile @@ -1,7 +1,11 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'centos/stream8' + config.vm.box = 'generic/centos8s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + sudo dnf config-manager --set-enabled powertools + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-9/Vagrantfile b/test/ansible/vagrant/centos-9/Vagrantfile index 412354f3d5d..1087225bec2 100644 --- a/test/ansible/vagrant/centos-9/Vagrantfile +++ b/test/ansible/vagrant/centos-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/centos9s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core + sudo dnf config-manager --set-enabled crb + SHELL end common = '../common' diff --git a/test/ansible/vagrant/common b/test/ansible/vagrant/common index adafa08c560..4bc237a7e5b 100644 --- a/test/ansible/vagrant/common +++ b/test/ansible/vagrant/common @@ -1,46 +1,75 @@ # vim: set ft=ruby: # frozen_string_literal: true +def find_ansible_cfg + path = Pathname.new(Dir.pwd) + until path.root? + ansible_cfg = path + 'ansible.cfg' + return path if ansible_cfg.exist? + path = path.parent + end + nil # return nil if not found +end + Vagrant.configure('2') do |config| config.vm.define 'crowdsec' + if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } + unless ENV['DB_BACKEND'] + $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" + exit 1 + end + end + + config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') + config.vm.synced_folder '.', '/vagrant', disabled: true + config.vm.provider :libvirt do |libvirt| libvirt.cpus = 1 libvirt.memory = 1536 end - config.vm.synced_folder '.', '/vagrant', disabled: true + path = find_ansible_cfg + if !path + puts "ansible.cfg not found" + end config.vm.provision 'ansible' do |ansible| # ansible.verbose = 'vvvv' - ansible.config_file = '../../ansible.cfg' - ansible.playbook = '../../run_all.yml' + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'run_all.yml').to_s + ansible.compatibility_mode = "2.0" end # same as above, to run the steps separately - # config.vm.provision 'ansible' do |provdep| - # provdep.config_file = '../../ansible.cfg' - # provdep.playbook = '../../provision_dependencies.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'provision_dependencies.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |provtest| - # provtest.config_file = '../../ansible.cfg' - # provtest.playbook = '../../provision_test_suite.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'provision_test_suite.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |preptest| - # preptest.config_file = '../../ansible.cfg' - # preptest.playbook = '../../install_binary_package.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'install_binary_package.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |preptest| - # preptest.config_file = '../../ansible.cfg' - # preptest.playbook = '../../prepare_tests.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'prepare_tests.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |runtests| - # runtests.config_file = '../../ansible.cfg' - # runtests.playbook = '../../run_tests.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'run_tests.yml').to_s + # ansible.compatibility_mode = "2.0" # end end diff --git a/test/ansible/vagrant/debian-10-buster/Vagrantfile b/test/ansible/vagrant/debian-10-buster/Vagrantfile index 2b1a4e2dac6..8d0a04eaf61 100644 --- a/test/ansible/vagrant/debian-10-buster/Vagrantfile +++ b/test/ansible/vagrant/debian-10-buster/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/buster64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile index 9166427cb3f..fbd2ca41eb4 100644 --- a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile +++ b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/bullseye64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile new file mode 100644 index 00000000000..68a33a94783 --- /dev/null +++ b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bookworm64' + config.vm.provision "shell", inline: <<-SHELL + # sudo apt install -y kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/debian-9-stretch/Vagrantfile b/test/ansible/vagrant/debian-9-stretch/Vagrantfile index 4c4e39cf734..d40b1640cb8 100644 --- a/test/ansible/vagrant/debian-9-stretch/Vagrantfile +++ b/test/ansible/vagrant/debian-9-stretch/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/stretch64' + config.vm.provision "shell", inline: <<-SHELL + sudo sed -i s/httpredir.debian.org/archive.debian.org/g /etc/apt/sources.list + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-testing/Vagrantfile b/test/ansible/vagrant/debian-testing/Vagrantfile index 5e3b68e5411..6cd2667a960 100644 --- a/test/ansible/vagrant/debian-testing/Vagrantfile +++ b/test/ansible/vagrant/debian-testing/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/testing64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile index 0d34ea126ce..975d39ea06a 100644 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile @@ -2,19 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/hardenedbsd13' - config.vm.define 'crowdsec' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 1 - libvirt.memory = 1536 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = '../../../ansible.cfg' - ansible.playbook = '../../../run_all.yml' - end + config.vm.provision "shell", inline: <<-SHELL + sudo pkg install python3 + SHELL end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap deleted file mode 100755 index 370b1b68e38..00000000000 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo pkg install python3 diff --git a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile new file mode 100644 index 00000000000..1a1fc342164 --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/openbsd6' + # config.vm.box_version = '4.2.16' + config.vm.provision "shell", inline: <<-SHELL + sudo pkg_add python py3-pip gcc openssl-1.0.2up3 gtar-1.34 + # sudo pkg_add -u + # sudo pkg_add kitty + SHELL +end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-6/skip b/test/ansible/vagrant/experimental/openbsd-6/skip new file mode 100755 index 00000000000..18d4c60f407 --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-6/skip @@ -0,0 +1,9 @@ +#!/bin/sh + +die() { + echo "$@" >&2 + exit 1 +} + +[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution" +exit 0 diff --git a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile index d7f9801e12b..1c11a94dcac 100644 --- a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile +++ b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile @@ -2,19 +2,13 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/openbsd7' - config.vm.define 'crowdsec' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 1 - libvirt.memory = 1536 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = '../../../ansible.cfg' - ansible.playbook = '../../../run_all.yml' - end + # config.vm.box_version = '4.2.16' + config.vm.provision "shell", inline: <<-SHELL + sudo pkg_add python-3.9.16 py3-pip gcc-11.2.0p3 openssl-3.0.8 gtar-1.34 + # sudo pkg_add -u + # sudo pkg_add kitty + SHELL end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-7/bootstrap b/test/ansible/vagrant/experimental/openbsd-7/bootstrap deleted file mode 100755 index 3b2480d3b8b..00000000000 --- a/test/ansible/vagrant/experimental/openbsd-7/bootstrap +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo pkg_add -u -sudo pkg_add python-3.9.13 py3-pip gcc-11.2.0p2 openssl-3.0.3p0 gtar-1.34 truncate-5.2.1 diff --git a/test/ansible/vagrant/fedora-33/Vagrantfile b/test/ansible/vagrant/fedora-33/Vagrantfile index 49b5ee990a0..df6f06944ae 100644 --- a/test/ansible/vagrant/fedora-33/Vagrantfile +++ b/test/ansible/vagrant/fedora-33/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/33-cloud-base" config.vm.box = 'generic/fedora33' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-34/Vagrantfile b/test/ansible/vagrant/fedora-34/Vagrantfile index 1d172c9c7fe..db2db8d0879 100644 --- a/test/ansible/vagrant/fedora-34/Vagrantfile +++ b/test/ansible/vagrant/fedora-34/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/34-cloud-base" config.vm.box = 'generic/fedora34' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-35/Vagrantfile b/test/ansible/vagrant/fedora-35/Vagrantfile index f1173076497..c526043405c 100644 --- a/test/ansible/vagrant/fedora-35/Vagrantfile +++ b/test/ansible/vagrant/fedora-35/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = 'fedora/35-cloud-base' config.vm.box = 'generic/fedora35' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-36/Vagrantfile b/test/ansible/vagrant/fedora-36/Vagrantfile index ef80f514c9e..0cd4fafd023 100644 --- a/test/ansible/vagrant/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/fedora-36/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/36-cloud-base" config.vm.box = 'generic/fedora36' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-37/Vagrantfile b/test/ansible/vagrant/fedora-37/Vagrantfile new file mode 100644 index 00000000000..0eb8978e349 --- /dev/null +++ b/test/ansible/vagrant/fedora-37/Vagrantfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/fedora37' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/fedora-38/Vagrantfile b/test/ansible/vagrant/fedora-38/Vagrantfile new file mode 100644 index 00000000000..0e5bf79edc7 --- /dev/null +++ b/test/ansible/vagrant/fedora-38/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = "fedora/38-cloud-base" + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/freebsd-12/Vagrantfile b/test/ansible/vagrant/freebsd-12/Vagrantfile index 33e6b473f7a..b500a64d99a 100644 --- a/test/ansible/vagrant/freebsd-12/Vagrantfile +++ b/test/ansible/vagrant/freebsd-12/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd12' + config.vm.provision "shell", inline: <<-SHELL + pkg install -y gtar + SHELL end common = '../common' diff --git a/test/ansible/vagrant/freebsd-13/Vagrantfile b/test/ansible/vagrant/freebsd-13/Vagrantfile index 851c04254b1..f416ad67793 100644 --- a/test/ansible/vagrant/freebsd-13/Vagrantfile +++ b/test/ansible/vagrant/freebsd-13/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd13' + config.vm.provision "shell", inline: <<-SHELL + pkg install -y gtar + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-7/Vagrantfile b/test/ansible/vagrant/oracle-7/Vagrantfile index 638a6123be2..bd435ca5436 100644 --- a/test/ansible/vagrant/oracle-7/Vagrantfile +++ b/test/ansible/vagrant/oracle-7/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle7' + config.vm.provision "shell", inline: <<-SHELL + sudo yum-config-manager --enable ol7_optional_latest + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-8/Vagrantfile b/test/ansible/vagrant/oracle-8/Vagrantfile index 425ad5e1661..6744ea87d4e 100644 --- a/test/ansible/vagrant/oracle-8/Vagrantfile +++ b/test/ansible/vagrant/oracle-8/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled ol8_codeready_builder + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-9/Vagrantfile b/test/ansible/vagrant/oracle-9/Vagrantfile index d4e3f618fba..8dcb4a13ba7 100644 --- a/test/ansible/vagrant/oracle-9/Vagrantfile +++ b/test/ansible/vagrant/oracle-9/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled ol9_codeready_builder + SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-8/Vagrantfile b/test/ansible/vagrant/rocky-8/Vagrantfile index c7315cc6718..27caf80c700 100644 --- a/test/ansible/vagrant/rocky-8/Vagrantfile +++ b/test/ansible/vagrant/rocky-8/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled powertools + sudo dnf -y install kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-9/Vagrantfile b/test/ansible/vagrant/rocky-9/Vagrantfile index 0adb3ab51b8..a4d06e4a9a3 100644 --- a/test/ansible/vagrant/rocky-9/Vagrantfile +++ b/test/ansible/vagrant/rocky-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled crb + sudo dnf -y install kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile index 86646ee7ab0..3af88099476 100644 --- a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu1604' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile index 70a77806bd6..5e25a2ee80c 100644 --- a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # the official boxes only supports virtualbox config.vm.box = 'generic/ubuntu1804' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile index 0006ae9267d..ea5b339078a 100644 --- a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2004' + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile index c0ccee54bf9..9e17f71fb6d 100644 --- a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile new file mode 100644 index 00000000000..6c15b0a1e30 --- /dev/null +++ b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/ubuntu2210' + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile new file mode 100644 index 00000000000..f40fb7bd59d --- /dev/null +++ b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'bento/ubuntu-23.04' + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/Vagrantfile b/test/ansible/vagrant/wizard/centos-8/Vagrantfile index 2df31a3920a..9db09a4ce01 100644 --- a/test/ansible/vagrant/wizard/centos-8/Vagrantfile +++ b/test/ansible/vagrant/wizard/centos-8/Vagrantfile @@ -1,38 +1,13 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'centos/stream8' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.box = 'generic/centos8s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + dnf config-manager --set-enabled powertools + # sudo dnf -y update + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/bootstrap b/test/ansible/vagrant/wizard/centos-8/bootstrap deleted file mode 100755 index b33ad9c881a..00000000000 --- a/test/ansible/vagrant/wizard/centos-8/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/common b/test/ansible/vagrant/wizard/common new file mode 100644 index 00000000000..be1820914c2 --- /dev/null +++ b/test/ansible/vagrant/wizard/common @@ -0,0 +1,67 @@ +# vim: set ft=ruby: +# frozen_string_literal: true + +def find_ansible_cfg + path = Pathname.new(Dir.pwd) + until path.root? + ansible_cfg = path + 'ansible.cfg' + return path if ansible_cfg.exist? + path = path.parent + end + nil # return nil if not found +end + +Vagrant.configure('2') do |config| + config.vm.define 'wizard' + + if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } + unless ENV['DB_BACKEND'] + $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" + exit 1 + end + end + + config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + path = find_ansible_cfg + if !path + puts "ansible.cfg not found" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'provision_dependencies.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'provision_test_suite.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'prepare_tests.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'debug_tools.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'run_wizard_tests.yml').to_s + ansible.compatibility_mode = "2.0" + end + +end diff --git a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile new file mode 100644 index 00000000000..3b10b312d0d --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/buster64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile new file mode 100644 index 00000000000..6dd7bb2fc9c --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bullseye64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile new file mode 100644 index 00000000000..5ccf234eb3e --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bookworm64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile deleted file mode 100644 index 1d6993a4e97..00000000000 --- a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bullseye64' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - -end diff --git a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap deleted file mode 100755 index 6a5df521a60..00000000000 --- a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile deleted file mode 100644 index d4380a94526..00000000000 --- a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/buster64' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - -end diff --git a/test/ansible/vagrant/wizard/debian-buster/bootstrap b/test/ansible/vagrant/wizard/debian-buster/bootstrap deleted file mode 100755 index 6a5df521a60..00000000000 --- a/test/ansible/vagrant/wizard/debian-buster/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile index 09ee8a39dc9..969a8e70c87 100644 --- a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile @@ -2,37 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'fedora/36-cloud-base' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + # sudo dnf -y update + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/fedora-36/bootstrap b/test/ansible/vagrant/wizard/fedora-36/bootstrap deleted file mode 100755 index b33ad9c881a..00000000000 --- a/test/ansible/vagrant/wizard/fedora-36/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile index 933dabab557..c13d2f9468e 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile @@ -2,42 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y aptitude kitty-terminfo + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap deleted file mode 100755 index 6a5df521a60..00000000000 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile index c3e8759aa6f..d0e2e3cdaa8 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile @@ -2,42 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2210' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y aptitude kitty-terminfo + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vars/go.yml b/test/ansible/vars/go.yml index 683b4fbbe80..929e412cd2c 100644 --- a/test/ansible/vars/go.yml +++ b/test/ansible/vars/go.yml @@ -1,5 +1,5 @@ # vim: set ft=yaml.ansible: --- -golang_version: "1.20.4" +golang_version: "1.20.5" golang_install_dir: "/opt/go/{{ golang_version }}" diff --git a/test/bats-detect/openresty-deb.bats b/test/bats-detect/openresty-deb.bats index c1e91949df3..0c8bc3c9a0b 100644 --- a/test/bats-detect/openresty-deb.bats +++ b/test/bats-detect/openresty-deb.bats @@ -37,8 +37,13 @@ setup() { run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/openresty.gpg < <(output) run -0 sudo tee <<< "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openresty.gpg] http://openresty.org/package/ubuntu $(lsb_release -sc) main" /etc/apt/sources.list.d/openresty.list else + release="$(lsb_release -sc)" + # Debian 12 package is not available as of 2023-07-3 + if [[ "$release" == "bookworm" ]]; then + release="bullseye" + fi run -0 sudo apt-key add - < <(output) - run -0 sudo tee <<< "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" /etc/apt/sources.list.d/openresty.list + run -0 sudo tee <<< "deb http://openresty.org/package/debian $release openresty" /etc/apt/sources.list.d/openresty.list fi run -0 deb-update run -0 deb-install openresty diff --git a/test/bats-detect/openresty-rpm.bats b/test/bats-detect/openresty-rpm.bats index 6fc0a8a0975..d4c3661bc76 100644 --- a/test/bats-detect/openresty-rpm.bats +++ b/test/bats-detect/openresty-rpm.bats @@ -34,10 +34,14 @@ setup() { run -0 rpm-install redhat-lsb-core if [[ "$(lsb_release -is)" == "Fedora" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/fedora/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" - elif [[ "$(lsb_release -is)" == "CentOS" ]]; then + elif [[ "$(lsb_release -is)" == CentOS* ]]; then # must match CentOSStream run -0 sudo curl -1sSLf "https://openresty.org/package/centos/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" fi - run -0 sudo dnf check-update + run sudo dnf check-update + # 0 = up to date, 100 = updates available + if [[ "$status" -ne 0 ]] && [[ "$status" -ne 100 ]]; then + fail "dnf check-update failed with status $status" + fi run -0 rpm-install openresty run -0 sudo systemctl enable openresty.service } diff --git a/test/bats.mk b/test/bats.mk index 65bb4a28608..4eb7abcbfb7 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -10,7 +10,7 @@ ifdef PACKAGE_TESTING INIT_BACKEND = systemd CONFIG_BACKEND = global else - # LOCAL_DIR will contain contains a local instance of crowdsec, complete with + # LOCAL_DIR contains a local instance of crowdsec, complete with # configuration and data LOCAL_DIR = $(TEST_DIR)/local BIN_DIR = $(LOCAL_DIR)/bin @@ -51,6 +51,7 @@ export CONFIG_BACKEND="$(CONFIG_BACKEND)" export PACKAGE_TESTING="$(PACKAGE_TESTING)" export TEST_COVERAGE="$(TEST_COVERAGE)" export GOCOVERDIR="$(TEST_DIR)/coverage" +export PATH="$(TEST_DIR)/tools:$(PATH)" endef bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 75b29033e6c..a1a2861f6df 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -151,9 +151,10 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -124 timeout 2s "${CROWDSEC}" + rune -1 timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition file found" + assert_stderr --partial "no acquisition file found" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -166,19 +167,56 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -124 timeout 2s "${CROWDSEC}" + rune -1 timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') - rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') mkdir -p "$ACQUIS_DIR" - touch "$ACQUIS_DIR"/foo.yaml + mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml rune -124 timeout 2s "${CROWDSEC}" + + # now, if foo.yaml is empty instead, there won't be valid datasources. + + cat /dev/null >"$ACQUIS_DIR"/foo.yaml + + rune -1 timeout 2s "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" +} + +@test "crowdsec (disabled datasources)" { + config_set '.common.log_media="stdout"' + + # a datasource cannot run - missing journalctl command + + ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') + mkdir -p "$ACQUIS_DIR" + cat >"$ACQUIS_DIR"/foo.yaml <<-EOT + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + EOT + + rune -124 timeout 2s env PATH='' "${CROWDSEC}" + #shellcheck disable=SC2016 + assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' + + # if all datasources are disabled, crowdsec should exit + + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + rm -f "$ACQUIS_YAML" + config_set '.crowdsec_service.acquisition_path=""' + + rune -1 timeout 2s env PATH='' "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } + diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index a01d936b787..ef825d2a8e6 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -101,9 +101,9 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) rune -0 cscli config show -o human - assert_line --regexp ".*- URL\s+: http://127.0.0.1:8080/" - assert_line --regexp ".*- Login\s+: githubciXXXXXXXXXXXXXXXXXXXXXXXX" - assert_line --regexp ".*- Credentials File\s+: .*/local_api_credentials.yaml" + assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" + assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX" + assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output) @@ -182,7 +182,7 @@ teardown() { @test "cscli - empty LAPI credentials file" { LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') - truncate -s 0 "${LOCAL_API_CREDENTIALS}" + : > "${LOCAL_API_CREDENTIALS}" rune -1 cscli lapi status assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'" diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index 7015f2c5dbd..f4c9f49e0f7 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -60,5 +60,5 @@ setup() { ONLINE_API_CREDENTIALS_YAML="$(config_get '.api.server.online_client.credentials_path')" rm "${ONLINE_API_CREDENTIALS_YAML}" rune -1 cscli capi status - assert_stderr --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" + assert_stderr --partial "local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" } diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index c216067f9e1..9640e333073 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -36,35 +36,35 @@ teardown() { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" } @test "duplicate notification config" { @@ -85,14 +85,14 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" } @test "config.yaml: missing .plugin_config section" { @@ -116,9 +116,9 @@ teardown() { assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" } -@test "unable to run local API: while reading plugin config" { +@test "unable to run plugin broker: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" } diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 3499f3e0ed8..bcb410de979 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -5,6 +5,9 @@ set -u setup_file() { load "../lib/setup_file.sh" + + TESTDATA="${BATS_TEST_DIRNAME}/testdata/90_decisions" + export TESTDATA } teardown_file() { @@ -56,8 +59,122 @@ teardown() { @test "cscli decisions list, incorrect parameters" { rune -1 cscli decisions list --until toto - assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"' + assert_stderr --partial 'unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\"' rune -1 cscli decisions list --until toto -o json rune -0 jq -c '[.level, .msg]' <(stderr | grep "^{") - assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]' + assert_output '["fatal","unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\""]' +} + +@test "cscli decisions import" { + # required input + rune -1 cscli decisions import + assert_stderr --partial 'required flag(s) \"input\" not set"' + + # unsupported format + rune -1 cscli decisions import -i - <<<'value\n5.6.7.8' --format xml + assert_stderr --partial "invalid format 'xml', expected one of 'json', 'csv', 'values'" + + # invalid defaults + rune -1 cscli decisions import --duration "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--duration cannot be empty" + rune -1 cscli decisions import --scope "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--scope cannot be empty" + rune -1 cscli decisions import --reason "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--reason cannot be empty" + rune -1 cscli decisions import --type "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--type cannot be empty" + + #---------- + # JSON + #---------- + + # import from file + rune -1 cscli decisions import -i "${TESTDATA}/json_decisions" + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + + rune -0 cscli decisions import -i "${TESTDATA}/decisions.json" + assert_stderr --partial "Parsing json" + assert_stderr --partial "Imported 5 decisions" + + # import from stdin + rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") --format json + assert_stderr --partial "Parsing json" + assert_stderr --partial "Imported 5 decisions" + + # invalid json + rune -1 cscli decisions import -i - <<<'{"blah":"blah"}' --format json + assert_stderr --partial 'Parsing json' + assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' + + # json with extra data + rune -1 cscli decisions import -i - <<<'{"values":"1.2.3.4","blah":"blah"}' --format json + assert_stderr --partial 'Parsing json' + assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' + + #---------- + # CSV + #---------- + + # import from file + rune -1 cscli decisions import -i "${TESTDATA}/csv_decisions" + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + + rune -0 cscli decisions import -i "${TESTDATA}/decisions.csv" + assert_stderr --partial 'Parsing csv' + assert_stderr --partial 'Imported 5 decisions' + + # import from stdin + rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") --format csv + assert_stderr --partial "Parsing csv" + assert_stderr --partial "Imported 5 decisions" + + # invalid csv + # XXX: improve validation + rune -0 cscli decisions import -i - <<<'value\n1.2.3.4,5.6.7.8' --format csv + assert_stderr --partial 'Parsing csv' + assert_stderr --partial "Imported 0 decisions" + + #---------- + # VALUES + #---------- + + # can use '-' as stdin + rune -0 cscli decisions import -i - --format values <<-EOT + 1.2.3.4 + 1.2.3.5 + 1.2.3.6 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + + rune -0 cscli decisions import -i - --format values <<-EOT + 10.2.3.4 + 10.2.3.5 + 10.2.3.6 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + + rune -1 cscli decisions import -i - --format values <<-EOT + whatever + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'API error: unable to create alerts: whatever: invalid ip address / range' + + #---------- + # Batch + #---------- + + rune -0 cscli decisions import -i - --format values --batch 2 --debug <<-EOT + 1.2.3.4 + 1.2.3.5 + 1.2.3.6 + EOT + assert_stderr --partial 'Processing chunk of 2 decisions' + assert_stderr --partial 'Processing chunk of 1 decisions' + assert_stderr --partial 'Imported 3 decisions' } diff --git a/test/bats/testdata/90_decisions/csv_decisions b/test/bats/testdata/90_decisions/csv_decisions new file mode 100644 index 00000000000..858654b6304 --- /dev/null +++ b/test/bats/testdata/90_decisions/csv_decisions @@ -0,0 +1,6 @@ +origin,scope,value,reason,type,duration +cscli,ip,1.6.11.16,manual import from csv,ban,1h +cscli,ip,2.7.12.17,manual import from csv,ban,1h +cscli,ip,3.8.13.18,manual import from csv,ban,1h +cscli,ip,4.9.14.19,manual import from csv,ban,1h +cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.csv b/test/bats/testdata/90_decisions/decisions.csv new file mode 100644 index 00000000000..858654b6304 --- /dev/null +++ b/test/bats/testdata/90_decisions/decisions.csv @@ -0,0 +1,6 @@ +origin,scope,value,reason,type,duration +cscli,ip,1.6.11.16,manual import from csv,ban,1h +cscli,ip,2.7.12.17,manual import from csv,ban,1h +cscli,ip,3.8.13.18,manual import from csv,ban,1h +cscli,ip,4.9.14.19,manual import from csv,ban,1h +cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.json b/test/bats/testdata/90_decisions/decisions.json new file mode 100644 index 00000000000..395458c9785 --- /dev/null +++ b/test/bats/testdata/90_decisions/decisions.json @@ -0,0 +1,42 @@ +[ + { + "origin": "cscli", + "scope": "ip", + "value": "1.6.11.16", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "2.7.12.17", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "3.8.13.18", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "4.9.14.19", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "5.10.15.20", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + } +] diff --git a/test/bats/testdata/90_decisions/json_decisions b/test/bats/testdata/90_decisions/json_decisions new file mode 100644 index 00000000000..395458c9785 --- /dev/null +++ b/test/bats/testdata/90_decisions/json_decisions @@ -0,0 +1,42 @@ +[ + { + "origin": "cscli", + "scope": "ip", + "value": "1.6.11.16", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "2.7.12.17", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "3.8.13.18", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "4.9.14.19", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "5.10.15.20", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + } +] diff --git a/test/bin/assert-crowdsec-not-running b/test/bin/assert-crowdsec-not-running index b545ebf0af3..3171287d0e6 100755 --- a/test/bin/assert-crowdsec-not-running +++ b/test/bin/assert-crowdsec-not-running @@ -1,8 +1,15 @@ #!/usr/bin/env bash is_crowdsec_running() { - # ignore processes in containers - PIDS=$(pgrep --ns $$ -x 'crowdsec') + case $(uname) in + "Linux") + # ignore processes in containers + PIDS=$(pgrep --ns $$ -x 'crowdsec') + ;; + *) + PIDS=$(pgrep -x 'crowdsec') + ;; + esac } # The process can be slow, especially on CI and during test coverage. diff --git a/test/bin/check-requirements b/test/bin/check-requirements index 351c0a01b13..0563eec01e4 100755 --- a/test/bin/check-requirements +++ b/test/bin/check-requirements @@ -54,47 +54,25 @@ check_pkill() { fi } -check_yq() { - # shellcheck disable=SC2016 - howto_install='You can install it with your favorite package manager (including snap) or with "go install github.com/mikefarah/yq/v4@latest" and add ~/go/bin to $PATH.' - if ! command -v yq >/dev/null; then - die "Missing required program 'yq'. ${howto_install}" - fi - if ! (yq --version | grep mikefarah >/dev/null); then - die "yq exists but it's not the one we need (mikefarah/yq). ${howto_install}" - fi -} - check_daemonizer() { if ! command -v daemonize >/dev/null; then die "missing required program 'daemonize' (package 'daemonize' or 'https://github.com/bmc/daemonize')" fi } -check_cfssl() { - # shellcheck disable=SC2016 - howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssl@latest" and add ~/go/bin to $PATH.' - if ! command -v cfssl >/dev/null; then - die "Missing required program 'cfssl'. ${howto_install}" - fi -} +echo "Checking requirements..." -check_cfssljson() { - # shellcheck disable=SC2016 - howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssljson@latest" and add ~/go/bin to $PATH.' - if ! command -v cfssljson >/dev/null; then - die "Missing required program 'cfssljson'. ${howto_install}" - fi -} +GOBIN=${TEST_DIR}/tools +export GOBIN +go install github.com/mikefarah/yq/v4@latest +go install github.com/cloudflare/cfssl/cmd/cfssl@master +go install github.com/cloudflare/cfssl/cmd/cfssljson@master check_bats_core check_curl check_daemonizer -check_cfssl -check_cfssljson check_jq check_nc check_base64 check_python3 -check_yq check_pkill diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 22a9bc303bc..a4231c98edb 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -127,6 +127,24 @@ is_db_sqlite() { } export -f is_db_sqlite +crowdsec_log() { + echo "$(config_get .common.log_dir)"/crowdsec.log +} +export -f crowdsec_log + +truncate_log() { + true > "$(crowdsec_log)" +} +export -f truncate_log + +assert_log() { + local oldout="${output:-}" + output="$(cat "$(crowdsec_log)")" + assert_output "$@" + output="${oldout}" +} +export -f assert_log + # Compare ignoring the key order, and allow "expected" without quoted identifiers. # Preserve the output variable in case the following commands require it. assert_json() { diff --git a/test/tools/.do-not-remove b/test/tools/.do-not-remove new file mode 100644 index 00000000000..3119281709f --- /dev/null +++ b/test/tools/.do-not-remove @@ -0,0 +1 @@ +this directory is populated by test dependencies, and is not checked in git From 343d22e7b3a846c6d7f181208286647d75107606 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 18:29:00 +0200 Subject: [PATCH 058/263] fix rules helpers --- pkg/waf/waf.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go index 9e674f01132..c1755bff36b 100644 --- a/pkg/waf/waf.go +++ b/pkg/waf/waf.go @@ -170,7 +170,9 @@ func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { //Ignore filter for on load ? if onLoadHook.Apply != nil { for exprIdx, applyExpr := range onLoadHook.Apply { - _, err := expr.Run(applyExpr, GetEnv()) + _, err := expr.Run(applyExpr, map[string]interface{}{ + "rules": collection, + }) if err != nil { w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) continue From da37b5566d9e887b6adf7c23066547202b49b954 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 18:35:35 +0200 Subject: [PATCH 059/263] update --- pkg/acquisition/modules/waf/waf.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 71be7464d35..2a5bdbc9173 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -574,10 +574,11 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if expTx != nil && len(expTx.MatchedRules()) > 0 { //if event was not instantiated after inband processing, do it now if evt == nil { - *evt, err = EventFromRequest(request) + tmpEvt, err := EventFromRequest(request) if err != nil { return fmt.Errorf("cannot create event from waap context : %w", err) } + evt = &tmpEvt } err = r.AccumulateTxToEvent(expTx, OutOfBand, evt) From 51295ef5774e78b6e7530b6dc05498f827d565e4 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 18:39:15 +0200 Subject: [PATCH 060/263] fix --- pkg/waf/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 42cd9d6d55d..07da7805851 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -37,11 +37,17 @@ func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interru } func (r *ResponseRequest) SetRemediation(remediation string) error { + if r.Interruption == nil { + return nil + } r.Interruption.Action = remediation return nil } func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { + if r.Interruption == nil { + return nil + } if r.Interruption.RuleID == ID { r.Interruption.Action = remediation } From 4332598cd1df96f7c87d5fe9f0190847c84b2269 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 18:44:32 +0200 Subject: [PATCH 061/263] add debug --- pkg/acquisition/modules/waf/utils.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index f3211459230..8a1edd6359e 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" ) func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { @@ -115,11 +116,12 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st return true }) - r.logger.Infof("variables addr in AccumulateTxToEvent: %p", tx.Variables()) //log.Infof("variables: %s", spew.Sdump(tx.Variables())) //log.Infof("tx variables: %+v", tx.Collection(variables.TX)) //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + for _, rule := range tx.MatchedRules() { + log.Infof("Matched rules ID: %+v", rule.Rule().ID()) if rule.Message() == "" { continue } From 353926ec918fa0144be62ab1baf78cdc17db51d8 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 31 Jul 2023 18:47:54 +0200 Subject: [PATCH 062/263] add debug --- pkg/acquisition/modules/waf/waf.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 2a5bdbc9173..56ffa5d1256 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -578,6 +578,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if err != nil { return fmt.Errorf("cannot create event from waap context : %w", err) } + r.logger.Infof("REAL MATCHED RULES: %+v", len(tmpEvt.Waap.MatchedRules)) evt = &tmpEvt } From cbf06c25fbc35ceb28a9b80304004db4c2ae1c90 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 1 Aug 2023 10:34:43 +0200 Subject: [PATCH 063/263] fix outofband evt generation --- pkg/acquisition/modules/waf/waf.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 56ffa5d1256..48fd8fd8beb 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -578,7 +578,6 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if err != nil { return fmt.Errorf("cannot create event from waap context : %w", err) } - r.logger.Infof("REAL MATCHED RULES: %+v", len(tmpEvt.Waap.MatchedRules)) evt = &tmpEvt } @@ -586,8 +585,13 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if err != nil { return fmt.Errorf("cannot convert transaction to event : %w", err) } - LogWaapEvent(evt) - r.outChan <- *evt + + // expTx.MatchedRules() returns also rules that set variables + // in evt.Waap.MatchedRules we have filtered those rules + if len(evt.Waap.MatchedRules) > 0 { + LogWaapEvent(evt) + r.outChan <- *evt + } } expTx.Close() //measure the full time spent in the WAF From 885c28309769ab92e299f80dd361303fc7c5bff4 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 1 Aug 2023 10:58:36 +0200 Subject: [PATCH 064/263] remove debug --- pkg/acquisition/modules/waf/utils.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 8a1edd6359e..be5ba81ccc5 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -10,7 +10,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" ) func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { @@ -121,7 +120,6 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) for _, rule := range tx.MatchedRules() { - log.Infof("Matched rules ID: %+v", rule.Rule().ID()) if rule.Message() == "" { continue } From 59e3d0dfcee9a56de1b71b6c511311c5eca5dd33 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 2 Aug 2023 11:43:49 +0200 Subject: [PATCH 065/263] distinct: return emtpy slice --- pkg/exprhelpers/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 8c7146342ac..f28aef84a36 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -180,7 +180,7 @@ func Distinct(params ...any) (any, error) { } array := params[0].([]interface{}) if array == nil { - return nil, nil + return []interface{}{}, nil } var exists map[any]bool = make(map[any]bool) From a4ee1e717e7a98ded071309673cbe96990292804 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 2 Aug 2023 11:47:35 +0200 Subject: [PATCH 066/263] try re2 for @rx operator --- pkg/acquisition/modules/waf/rx_operator.go | 94 ++++++++++++++++++++++ pkg/acquisition/modules/waf/waf.go | 2 + 2 files changed, 96 insertions(+) create mode 100644 pkg/acquisition/modules/waf/rx_operator.go diff --git a/pkg/acquisition/modules/waf/rx_operator.go b/pkg/acquisition/modules/waf/rx_operator.go new file mode 100644 index 00000000000..f1335c7e3aa --- /dev/null +++ b/pkg/acquisition/modules/waf/rx_operator.go @@ -0,0 +1,94 @@ +package wafacquisition + +import ( + "fmt" + "strconv" + "unicode/utf8" + + "github.com/crowdsecurity/coraza/v3/experimental/plugins" + "github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes" + "github.com/wasilibs/go-re2" + "github.com/wasilibs/go-re2/experimental" +) + +type rx struct { + re *re2.Regexp +} + +var _ plugintypes.Operator = (*rx)(nil) + +func newRX(options plugintypes.OperatorOptions) (plugintypes.Operator, error) { + // (?sm) enables multiline mode which makes 942522-7 work, see + // - https://stackoverflow.com/a/27680233 + // - https://groups.google.com/g/golang-nuts/c/jiVdamGFU9E + data := fmt.Sprintf("(?sm)%s", options.Arguments) + + var re *re2.Regexp + var err error + + if matchesArbitraryBytes(data) { + re, err = experimental.CompileLatin1(data) + } else { + re, err = re2.Compile(data) + } + if err != nil { + return nil, err + } + return &rx{re: re}, nil +} + +func (o *rx) Evaluate(tx plugintypes.TransactionState, value string) bool { + if tx.Capturing() { + match := o.re.FindStringSubmatch(value) + if len(match) == 0 { + return false + } + for i, c := range match { + if i == 9 { + return true + } + tx.CaptureField(i, c) + } + return true + } else { + return o.re.MatchString(value) + } +} + +// RegisterRX registers the rx operator using a WASI implementation instead of Go. +func RegisterRX() { + plugins.RegisterOperator("rx", newRX) +} + +// matchesArbitraryBytes checks for control sequences for byte matches in the expression. +// If the sequences are not valid utf8, it returns true. +func matchesArbitraryBytes(expr string) bool { + decoded := make([]byte, 0, len(expr)) + for i := 0; i < len(expr); i++ { + c := expr[i] + if c != '\\' { + decoded = append(decoded, c) + continue + } + if i+3 >= len(expr) { + decoded = append(decoded, expr[i:]...) + break + } + if expr[i+1] != 'x' { + decoded = append(decoded, expr[i]) + continue + } + + v, mb, _, err := strconv.UnquoteChar(expr[i:], 0) + if err != nil || mb { + // Wasn't a byte escape sequence, shouldn't happen in practice. + decoded = append(decoded, expr[i]) + continue + } + + decoded = append(decoded, byte(v)) + i += 3 + } + + return !utf8.Valid(decoded) +} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 48fd8fd8beb..ead8a7ef031 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -176,6 +176,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { Handler: w.mux, } + RegisterRX() + ruleLoader := waf.NewWafRuleLoader() rulesCollections, err := ruleLoader.LoadWafRules() From 4846701ed5a57634343871db741506918cecd69c Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 21 Aug 2023 15:34:18 +0200 Subject: [PATCH 067/263] logging --- pkg/acquisition/modules/waf/utils.go | 32 ++++++++++++++++++++++------ pkg/acquisition/modules/waf/waf.go | 8 +++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index be5ba81ccc5..a31c39bd975 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" ) func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { @@ -47,12 +48,26 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { return evt, nil } -func LogWaapEvent(evt *types.Event) { - /*log.WithFields(log.Fields{ - "module": "waf", - "source": evt.Parsed["source_ip"], - "target_uri": evt.Parsed["target_uri"], - }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs())*/ +func LogWaapEvent(evt *types.Event, logger *log.Entry) { + req := evt.Parsed["target_uri"] + if len(req) > 12 { + req = req[:10] + ".." + } + + if evt.Parsed["interrupted"] == "true" { + logger.WithFields(log.Fields{ + "module": "waf", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + } else { + logger.WithFields(log.Fields{ + "module": "waf", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Debugf("%s triggerd non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + } + //log.Infof("%s", evt.Waap) } @@ -63,6 +78,11 @@ func LogWaapEvent(evt *types.Event) { */ +// func LogWaapEvent(evt *types.Event) error { + +// return nil +// } + func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { //log.Infof("tx addr: %p", tx) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index ead8a7ef031..ec247f68a07 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -543,6 +543,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } } + logged := false //measure the full time spent in the WAF elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) @@ -558,7 +559,8 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { if err != nil { return fmt.Errorf("cannot convert transaction to event : %w", err) } - LogWaapEvent(evt) + LogWaapEvent(evt, r.logger) + logged = true r.outChan <- *evt } expTx.Close() @@ -591,7 +593,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // expTx.MatchedRules() returns also rules that set variables // in evt.Waap.MatchedRules we have filtered those rules if len(evt.Waap.MatchedRules) > 0 { - LogWaapEvent(evt) + if !logged { + LogWaapEvent(evt, r.logger) + } r.outChan <- *evt } } From e0bd4dc9280b1e19a0a068d1ee5f0177b8ff451d Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Thu, 24 Aug 2023 12:11:44 +0200 Subject: [PATCH 068/263] fix linter --- pkg/acquisition/modules/waf/waf.go | 8 +++----- pkg/waf/request.go | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index ec247f68a07..341e2e5c83a 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -61,8 +61,6 @@ type WafSource struct { outChan chan types.Event InChan chan waf.ParsedRequest - inBandWaf coraza.WAF - outOfBandWaf coraza.WAF RulesCollections []*waf.WafRulesCollection WafRunners []WafRunner @@ -464,7 +462,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { switch t := res.(type) { case bool: - if t == false { + if !t { log.Infof("filter didnt match") continue } @@ -518,7 +516,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { switch t := res.(type) { case bool: - if t == false { + if !t { continue } default: @@ -569,6 +567,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // Process outBand outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) expTx = outBandTx.(experimental.FullTransaction) + in, expTx, err = r.processReqWithEngine(expTx, request, OutOfBand) if err != nil { //things went south r.logger.Errorf("Error while processing request : %s", err) @@ -655,5 +654,4 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { rw.Write(body) } - return } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 07da7805851..bed7a6ef90a 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -2,7 +2,7 @@ package waf import ( "fmt" - "io/ioutil" + "io" "net/http" "net/url" @@ -78,12 +78,11 @@ type ParsedRequest struct { } func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { - var body []byte var err error + body := make([]byte, 0) if r.Body != nil { - body = make([]byte, 0) - body, err = ioutil.ReadAll(r.Body) + body, err = io.ReadAll(r.Body) if err != nil { return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) } From 0379574b14b8b14eb117bd872d415f09913a0587 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Thu, 31 Aug 2023 11:07:51 +0200 Subject: [PATCH 069/263] support SSL for waf --- pkg/acquisition/modules/waf/waf.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 341e2e5c83a..7e9ce70e931 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -45,6 +45,8 @@ type WafRunner struct { type WafSourceConfig struct { ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` + CertFilePath string `yaml:"cert_file"` + KeyFilePath string `yaml:"key_file"` Path string `yaml:"path"` WafRoutines int `yaml:"waf_routines"` Debug bool `yaml:"debug"` @@ -315,7 +317,13 @@ func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) err w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) t.Go(func() error { - err := w.server.ListenAndServe() + var err error + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { + err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) + } else { + err = w.server.ListenAndServe() + } + if err != nil && err != http.ErrServerClosed { return errors.Wrap(err, "WAF server failed") } From 24d2c264a794c17445bba8fd219aabe649c04ab7 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 5 Sep 2023 17:56:02 +0200 Subject: [PATCH 070/263] clarify logging if triggering inband or outofband rules --- pkg/acquisition/modules/waf/utils.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index a31c39bd975..a31b632b2c7 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -54,12 +54,18 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { req = req[:10] + ".." } - if evt.Parsed["interrupted"] == "true" { + if evt.Meta["waap_interrupted"] == "true" { logger.WithFields(log.Fields{ "module": "waf", "source": evt.Parsed["source_ip"], "target_uri": req, }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + } else if evt.Parsed["outofband_interrupted"] == "true" { + logger.WithFields(log.Fields{ + "module": "waf", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) } else { logger.WithFields(log.Fields{ "module": "waf", @@ -68,7 +74,6 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { }).Debugf("%s triggerd non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) } - //log.Infof("%s", evt.Waap) } /* @@ -92,13 +97,15 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st if evt.Meta == nil { evt.Meta = map[string]string{} } - evt.Parsed["interrupted"] = "true" - evt.Parsed["action"] = tx.Interruption().Action - - //log.Infof("action: %s", tx.Interruption().Action) - - evt.Meta["waap_interrupted"] = "1" - evt.Meta["waap_action"] = tx.Interruption().Action + if kind == InBand { + evt.Meta["waap_interrupted"] = "true" + evt.Meta["waap_action"] = tx.Interruption().Action + evt.Parsed["inband_interrupted"] = "true" + evt.Parsed["inband_action"] = tx.Interruption().Action + } else { + evt.Parsed["outofband_interrupted"] = "true" + evt.Parsed["outofband_action"] = tx.Interruption().Action + } } if evt.Waap.Vars == nil { From 4e26e23725a94fdb6d139a5ef60e3baf0fd30e8b Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Mon, 11 Sep 2023 10:35:14 +0200 Subject: [PATCH 071/263] Waap config (#2460) * revamp wip --- pkg/acquisition/acquisition.go | 5 +- pkg/acquisition/modules/waf/metrics.go | 46 ++ pkg/acquisition/modules/waf/utils.go | 21 +- pkg/acquisition/modules/waf/waap.go | 274 +++++++++ pkg/acquisition/modules/waf/waap_runner.go | 67 +++ pkg/acquisition/modules/waf/waf.go | 665 --------------------- pkg/waf/env.go | 24 +- pkg/waf/waap.go | 237 ++++++++ pkg/waf/waap_rules_collection.go | 34 ++ pkg/waf/waf.go | 204 ------- pkg/waf/waf_rules_collection.go | 37 -- 11 files changed, 681 insertions(+), 933 deletions(-) create mode 100644 pkg/acquisition/modules/waf/metrics.go create mode 100644 pkg/acquisition/modules/waf/waap.go create mode 100644 pkg/acquisition/modules/waf/waap_runner.go delete mode 100644 pkg/acquisition/modules/waf/waf.go create mode 100644 pkg/waf/waap.go create mode 100644 pkg/waf/waap_rules_collection.go delete mode 100644 pkg/waf/waf.go delete mode 100644 pkg/waf/waf_rules_collection.go diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 42020accc7b..b37f9f985d4 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -37,7 +37,7 @@ import ( type DataSourceUnavailableError struct { Name string - Err error + Err error } func (e *DataSourceUnavailableError) Error() string { @@ -48,7 +48,6 @@ func (e *DataSourceUnavailableError) Unwrap() error { return e.Err } - // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -76,7 +75,7 @@ var AcquisitionSources = map[string]func() DataSource{ "kafka": func() DataSource { return &kafkaacquisition.KafkaSource{} }, "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, - "waf": func() DataSource { return &wafacquisition.WafSource{} }, + "waf": func() DataSource { return &wafacquisition.WaapSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/modules/waf/metrics.go b/pkg/acquisition/modules/waf/metrics.go new file mode 100644 index 00000000000..db9747250e4 --- /dev/null +++ b/pkg/acquisition/modules/waf/metrics.go @@ -0,0 +1,46 @@ +package wafacquisition + +import "github.com/prometheus/client_golang/prometheus" + +var WafGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband WAF.", + Name: "cs_waf_inband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_outband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_reqs_total", + Help: "Total events processed by the WAF.", + }, + []string{"source"}, +) + +var WafRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_rule_hits", + Help: "Count of triggered rule, by rule_id and type (inband/outofband).", + }, + []string{"rule_id", "type"}, +) diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index a31b632b2c7..4bae89b458b 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -88,12 +88,9 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { // return nil // } -func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { +func AccumulateTxToEvent(logger log.Entry, tx experimental.FullTransaction, kind string, evt *types.Event, wr *waf.WaapRuntimeConfig) error { - //log.Infof("tx addr: %p", tx) if tx.IsInterrupted() { - //r.logger.Infof("interrupted() = %t", tx.IsInterrupted()) - //r.logger.Infof("interrupted.action = %s", tx.Interruption().Action) if evt.Meta == nil { evt.Meta = map[string]string{} } @@ -112,12 +109,6 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st evt.Waap.Vars = map[string]string{} } - // collectionsToKeep := []string{ - // "toto", - // "TX.allowed_methods", - // "TX.*_score", - // } - tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { for _, variable := range col.FindAll() { key := "" @@ -129,23 +120,19 @@ func (r *WafRunner) AccumulateTxToEvent(tx experimental.FullTransaction, kind st if variable.Value() == "" { continue } - for _, collectionToKeep := range r.VariablesTracking { + for _, collectionToKeep := range wr.CompiledVariablesTracking { match := collectionToKeep.MatchString(key) if match { evt.Waap.Vars[key] = variable.Value() - r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } else { - r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) + logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) } } } return true }) - //log.Infof("variables: %s", spew.Sdump(tx.Variables())) - //log.Infof("tx variables: %+v", tx.Collection(variables.TX)) - //log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) - for _, rule := range tx.MatchedRules() { if rule.Message() == "" { continue diff --git a/pkg/acquisition/modules/waf/waap.go b/pkg/acquisition/modules/waf/waap.go new file mode 100644 index 00000000000..f34c1bd86fd --- /dev/null +++ b/pkg/acquisition/modules/waf/waap.go @@ -0,0 +1,274 @@ +package wafacquisition + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/crowdsec/pkg/waf" + "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" +) + +const ( + InBand = "inband" + OutOfBand = "outofband" +) + +// configuration structure of the acquis for the Waap +type WaapSourceConfig struct { + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + CertFilePath string `yaml:"cert_file"` + KeyFilePath string `yaml:"key_file"` + Path string `yaml:"path"` + Routines int `yaml:"routines"` + Debug bool `yaml:"debug"` + WaapConfig string `yaml:"waap_config"` + configuration.DataSourceCommonCfg `yaml:",inline"` +} + +// runtime structure of WaapSourceConfig +type WaapSource struct { + config WaapSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + InChan chan waf.ParsedRequest + WaapRuntime *waf.WaapRuntimeConfig + + WaapRunners []WaapRunner //one for each go-routine +} + +func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { + err := yaml.UnmarshalStrict(yamlConfig, wc.config) + if err != nil { + return errors.Wrap(err, "Cannot parse waf configuration") + } + + if wc.config.ListenAddr == "" { + return fmt.Errorf("listen_addr cannot be empty") + } + + if wc.config.ListenPort == 0 { + return fmt.Errorf("listen_port cannot be empty") + } + + if wc.config.Path == "" { + return fmt.Errorf("path cannot be empty") + } + + if wc.config.Path[0] != '/' { + wc.config.Path = "/" + wc.config.Path + } + + if wc.config.Mode == "" { + wc.config.Mode = configuration.TAIL_MODE + } + + // always have at least one waf routine + if wc.config.Routines == 0 { + wc.config.Routines = 1 + } + return nil +} + +func (w *WaapSource) GetMetrics() []prometheus.Collector { + return nil +} + +func (w *WaapSource) GetAggregMetrics() []prometheus.Collector { + return nil +} + +func logError(error corazatypes.MatchedRule) { + msg := error.ErrorLog() + log.Infof("[logError][%s] %s", error.Rule().Severity(), msg) +} + +func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { + wc := WaapSourceConfig{} + err := w.UnmarshalConfig(yamlConfig) + if err != nil { + return errors.Wrap(err, "unable to parse waf configuration") + } + w.logger = logger + w.config = wc + + w.logger.Tracef("WAF configuration: %+v", w.config) + + w.addr = fmt.Sprintf("%s:%d", wc.ListenAddr, wc.ListenPort) + + w.mux = http.NewServeMux() + + w.server = &http.Server{ + Addr: w.addr, + Handler: w.mux, + } + + w.InChan = make(chan waf.ParsedRequest) + w.WaapRunners = make([]WaapRunner, wc.Routines) + + for nbRoutine := 0; nbRoutine < wc.Routines; nbRoutine++ { + + wafUUID := uuid.New().String() + wafLogger := &log.Entry{} + + //configure logger + if wc.Debug { + var clog = log.New() + if err := types.ConfigureLogger(clog); err != nil { + log.Fatalf("While creating bucket-specific logger : %s", err) + } + clog.SetLevel(log.DebugLevel) + wafLogger = clog.WithFields(log.Fields{ + "uuid": wafUUID, + }) + } else { + wafLogger = log.WithFields(log.Fields{ + "uuid": wafUUID, + }) + } + + runner := WaapRunner{ + inChan: w.InChan, + UUID: wafUUID, + logger: wafLogger, + } + w.WaapRunners[nbRoutine] = runner + //most likely missign somethign here to actually start the runner :) + } + + w.logger.Infof("Created %d waf runners", len(w.WaapRunners)) + + //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec + w.mux.HandleFunc(w.config.Path, w.waapHandler) + + return nil +} + +func (w *WaapSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WaapSource) GetMode() string { + return w.config.Mode +} + +func (w *WaapSource) GetName() string { + return "waf" +} + +func (w *WaapSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("WAF datasource does not support command line acquisition") +} + +func (w *WaapSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { + w.outChan = out + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/waf/live") + + w.logger.Infof("%d waf runner to start", len(w.WaapRunners)) + for _, runner := range w.WaapRunners { + runner := runner + runner.outChan = out + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/waf/live/runner") + return runner.Run(t) + }) + } + + w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + t.Go(func() error { + var err error + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { + err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) + } else { + err = w.server.ListenAndServe() + } + + if err != nil && err != http.ErrServerClosed { + return errors.Wrap(err, "WAF server failed") + } + return nil + }) + <-t.Dying() + w.logger.Infof("Stopping WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + w.server.Shutdown(context.TODO()) + return nil + }) + return nil +} + +func (w *WaapSource) CanRun() error { + return nil +} + +func (w *WaapSource) GetUuid() string { + return w.config.UniqueId +} + +func (w *WaapSource) Dump() interface{} { + return w +} + +type BodyResponse struct { + Action string `json:"action"` +} + +// should this be in the runner ? +func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { + // parse the request only once + parsedRequest, err := waf.NewParsedRequestFromRequest(r) + if err != nil { + log.Errorf("%s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + w.InChan <- parsedRequest + + message := <-parsedRequest.ResponseChannel + + if message.Err != nil { + log.Errorf("Error while processing InBAND: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + + //here we must rely on WaapRuntimeConfig to know what to do + if message.Interruption != nil { + rw.WriteHeader(http.StatusForbidden) + action := message.Interruption.Action + if action == "deny" { // bouncers understand "ban" and not "deny" + action = "ban" + } + body, err := json.Marshal(BodyResponse{Action: action}) + if err != nil { + log.Errorf("unable to build response: %s", err) + } else { + rw.Write(body) + } + return + } + + rw.WriteHeader(http.StatusOK) + body, err := json.Marshal(BodyResponse{Action: "allow"}) + if err != nil { + log.Errorf("unable to marshal response: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + } else { + rw.Write(body) + } + +} diff --git a/pkg/acquisition/modules/waf/waap_runner.go b/pkg/acquisition/modules/waf/waap_runner.go new file mode 100644 index 00000000000..979a958b9d3 --- /dev/null +++ b/pkg/acquisition/modules/waf/waap_runner.go @@ -0,0 +1,67 @@ +package wafacquisition + +import ( + "time" + + "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/crowdsec/pkg/waf" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" +) + +// that's the runtime structure of the WAAP as seen from the acquis +type WaapRunner struct { + outChan chan types.Event + inChan chan waf.ParsedRequest + UUID string + WaapRuntime *waf.WaapRuntimeConfig //this holds the actual waap runtime config, rules, remediations, hooks etc. + WaapInbandEngine coraza.WAF + WaapOutbandEngine coraza.WAF + logger *log.Entry +} + +func (r *WaapRunner) Run(t *tomb.Tomb) error { + r.logger.Infof("Waap Runner ready to process event") + for { + select { + case <-t.Dying(): + r.logger.Infof("Waf Runner is dying") + return nil + case request := <-r.inChan: + r.logger.Infof("Requests handled by runner %s", request.UUID) + + //tx := waf.NewExtendedTransaction(r.WaapInbandEngine, r.UUID) + WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() + //measure the time spent in the WAF + startParsing := time.Now() + + //pre eval (expr) rules + err := r.WaapRuntime.ProcessPreEvalRules(request) + if err != nil { + r.logger.Errorf("unable to process PreEval rules: %s", err) + continue + } + + //inband WAAP rules + interrupt, err := r.WaapRuntime.ProcessInBandRules(request) + elapsed := time.Since(startParsing) + WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) + + //generate reponse for the remediation component, based on the WAAP config + inband rules evaluation + //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation + response := waf.NewResponseRequest(r.WaapRuntime.InBandTx.Tx, interrupt, request.UUID, err) + + err = r.WaapRuntime.ProcessOnMatchRules(request, response) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + continue + } + + // send back the result to the HTTP handler for the InBand part + request.ResponseChannel <- response + + } + } +} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go deleted file mode 100644 index 7e9ce70e931..00000000000 --- a/pkg/acquisition/modules/waf/waf.go +++ /dev/null @@ -1,665 +0,0 @@ -package wafacquisition - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "regexp" - "strings" - "time" - - "github.com/antonmedv/expr" - "github.com/crowdsecurity/coraza/v3" - "github.com/crowdsecurity/coraza/v3/experimental" - corazatypes "github.com/crowdsecurity/coraza/v3/types" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/crowdsecurity/go-cs-lib/trace" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - "gopkg.in/yaml.v2" -) - -const ( - InBand = "inband" - OutOfBand = "outofband" -) - -type WafRunner struct { - outChan chan types.Event - inChan chan waf.ParsedRequest - inBandWaf coraza.WAF - outOfBandWaf coraza.WAF - UUID string - RulesCollections []*waf.WafRulesCollection - logger *log.Entry - VariablesTracking []*regexp.Regexp -} - -type WafSourceConfig struct { - ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` - CertFilePath string `yaml:"cert_file"` - KeyFilePath string `yaml:"key_file"` - Path string `yaml:"path"` - WafRoutines int `yaml:"waf_routines"` - Debug bool `yaml:"debug"` - VariablesTracking []string `yaml:"variables_tracking"` - configuration.DataSourceCommonCfg `yaml:",inline"` -} - -type WafSource struct { - config WafSourceConfig - logger *log.Entry - mux *http.ServeMux - server *http.Server - addr string - outChan chan types.Event - InChan chan waf.ParsedRequest - - RulesCollections []*waf.WafRulesCollection - - WafRunners []WafRunner -} - -var WafGlobalParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the WAF.", - Name: "cs_waf_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafInbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the inband WAF.", - Name: "cs_waf_inband_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafOutbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the WAF.", - Name: "cs_waf_outband_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafReqCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_waf_reqs_total", - Help: "Total events processed by the WAF.", - }, - []string{"source"}, -) - -var WafRuleHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_waf_rule_hits", - Help: "Count of triggered rule, by rule_id and type (inband/outofband).", - }, - []string{"rule_id", "type"}, -) - -func (w *WafSource) GetMetrics() []prometheus.Collector { - return nil -} - -func (w *WafSource) GetAggregMetrics() []prometheus.Collector { - return nil -} - -func (w *WafSource) UnmarshalConfig(yamlConfig []byte) error { - wafConfig := WafSourceConfig{} - err := yaml.UnmarshalStrict(yamlConfig, &wafConfig) - if err != nil { - return errors.Wrap(err, "Cannot parse waf configuration") - } - - w.config = wafConfig - - if w.config.ListenAddr == "" { - return fmt.Errorf("listen_addr cannot be empty") - } - - if w.config.ListenPort == 0 { - return fmt.Errorf("listen_port cannot be empty") - } - - //FIXME: is that really needed ? - if w.config.Path == "" { - return fmt.Errorf("path cannot be empty") - } - - if w.config.Path[0] != '/' { - w.config.Path = "/" + w.config.Path - } - - if w.config.Mode == "" { - w.config.Mode = configuration.TAIL_MODE - } - - return nil -} - -func logError(error corazatypes.MatchedRule) { - msg := error.ErrorLog() - log.Infof("[logError][%s] %s", error.Rule().Severity(), msg) -} - -func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { - err := w.UnmarshalConfig(yamlConfig) - if err != nil { - return errors.Wrap(err, "Cannot parse waf configuration") - } - - w.logger = logger - - w.logger.Tracef("WAF configuration: %+v", w.config) - - w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) - - w.mux = http.NewServeMux() - - w.server = &http.Server{ - Addr: w.addr, - Handler: w.mux, - } - - RegisterRX() - - ruleLoader := waf.NewWafRuleLoader() - - rulesCollections, err := ruleLoader.LoadWafRules() - if err != nil { - return fmt.Errorf("cannot load WAF rules: %w", err) - } - - w.RulesCollections = rulesCollections - - var inBandRules string - var outOfBandRules string - - //spew.Dump(rulesCollections) - - for _, collection := range rulesCollections { - if !collection.OutOfBand { - inBandRules += collection.String() + "\n" - } else { - outOfBandRules += collection.String() + "\n" - } - } - - w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) - - //w.logger.Infof("Loading rules %+v", inBandRules) - - fs := os.DirFS(ruleLoader.Datadir) - // always have at least one waf routine - if w.config.WafRoutines == 0 { - w.config.WafRoutines = 1 - } - - w.InChan = make(chan waf.ParsedRequest) - w.logger.Infof("w.InChan creation: %p", w.InChan) - w.WafRunners = make([]WafRunner, w.config.WafRoutines) - for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ { - w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n"))) - - //in-band waf : kill on sight - inbandwaf, err := coraza.NewWAF( - coraza.NewWAFConfig(). - //WithErrorCallback(logError). - WithDirectives(inBandRules).WithRootFS(fs), - ) - - if err != nil { - return errors.Wrap(err, "Cannot create WAF") - } - w.logger.Infof("Loading %d out-of-band rules", len(strings.Split(outOfBandRules, "\n"))) - //out-of-band waf : log only - outofbandwaf, err := coraza.NewWAF( - coraza.NewWAFConfig(). - //WithErrorCallback(logError). - WithDirectives(outOfBandRules).WithRootFS(fs), - ) - - if err != nil { - return errors.Wrap(err, "Cannot create WAF") - } - wafUUID := uuid.New().String() - wafLogger := &log.Entry{} - if w.config.Debug { - var clog = log.New() - if err := types.ConfigureLogger(clog); err != nil { - log.Fatalf("While creating bucket-specific logger : %s", err) - } - clog.SetLevel(log.DebugLevel) - wafLogger = clog.WithFields(log.Fields{ - "uuid": wafUUID, - }) - } else { - wafLogger = log.WithFields(log.Fields{ - "uuid": wafUUID, - }) - } - - var compiledVariableRules []*regexp.Regexp - - for _, variable := range w.config.VariablesTracking { - compiledVariableRule, err := regexp.Compile(variable) - if err != nil { - return fmt.Errorf("cannot compile variable regexp %s: %w", variable, err) - } - compiledVariableRules = append(compiledVariableRules, compiledVariableRule) - } - - runner := WafRunner{ - outOfBandWaf: outofbandwaf, - inBandWaf: inbandwaf, - inChan: w.InChan, - UUID: wafUUID, - RulesCollections: rulesCollections, - logger: wafLogger, - VariablesTracking: compiledVariableRules, - } - w.WafRunners[nbRoutine] = runner - } - - w.logger.Infof("Created %d waf runners", len(w.WafRunners)) - - //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec - w.mux.HandleFunc(w.config.Path, w.wafHandler) - - return nil -} - -func (w *WafSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { - return fmt.Errorf("WAF datasource does not support command line acquisition") -} - -func (w *WafSource) GetMode() string { - return w.config.Mode -} - -func (w *WafSource) GetName() string { - return "waf" -} - -func (w *WafSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { - return fmt.Errorf("WAF datasource does not support command line acquisition") -} - -func (w *WafSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { - w.outChan = out - t.Go(func() error { - defer trace.CatchPanic("crowdsec/acquis/waf/live") - - w.logger.Infof("%d waf runner to start", len(w.WafRunners)) - for _, runner := range w.WafRunners { - runner := runner - runner.outChan = out - t.Go(func() error { - defer trace.CatchPanic("crowdsec/acquis/waf/live/runner") - return runner.Run(t) - }) - } - - w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) - t.Go(func() error { - var err error - if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { - err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) - } else { - err = w.server.ListenAndServe() - } - - if err != nil && err != http.ErrServerClosed { - return errors.Wrap(err, "WAF server failed") - } - return nil - }) - <-t.Dying() - w.logger.Infof("Stopping WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) - w.server.Shutdown(context.TODO()) - return nil - }) - return nil -} - -func (w *WafSource) CanRun() error { - return nil -} - -func (w *WafSource) GetUuid() string { - return w.config.UniqueId -} - -func (w *WafSource) Dump() interface{} { - return w -} - -func (r *WafRunner) processReqWithEngine(tx experimental.FullTransaction, parsedRequest waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) { - var in *corazatypes.Interruption - if tx.IsRuleEngineOff() { - r.logger.Printf("engine is off") - return nil, nil, nil - } - - defer func() { - tx.ProcessLogging() - //Dont close the transaction here: we still need access to the variables afterwards - //tx.Close() - }() - - //this method is not exported by coraza, so we have to do it ourselves. - //ideally, this would be dealt with by expr code, and we provide helpers to manipulate the transaction object?\ - //var txx experimental.FullTransaction - - //txx := experimental.ToFullInterface(tx) - //txx = tx.(experimental.FullTransaction) - //txx.RemoveRuleByID(1) - tx.ProcessConnection(parsedRequest.ClientIP, 0, "", 0) - - //tx.ProcessURI(parsedRequest.URL.String(), parsedRequest.Method, parsedRequest.Proto) //FIXME: get it from the headers - tx.ProcessURI(parsedRequest.URI, parsedRequest.Method, parsedRequest.Proto) //FIXME: get it from the headers - - for k, vr := range parsedRequest.Headers { - for _, v := range vr { - tx.AddRequestHeader(k, v) - } - } - - if parsedRequest.ClientHost != "" { - tx.AddRequestHeader("Host", parsedRequest.ClientHost) - // This connector relies on the host header (now host field) to populate ServerName - tx.SetServerName(parsedRequest.ClientHost) - } - - if parsedRequest.TransferEncoding != nil { - tx.AddRequestHeader("Transfer-Encoding", parsedRequest.TransferEncoding[0]) - } - - in = tx.ProcessRequestHeaders() - - //spew.Dump(in) - //spew.Dump(tx.MatchedRules()) - - /*for _, rule := range tx.MatchedRules() { - log.Infof("Rule %d disruptive: %t", rule.Rule().ID(), rule.Disruptive()) - if rule.Message() == "" { - continue - } - }*/ - - //if we're inband, we should stop here, but for outofband go to the end - if in != nil && wafType == InBand { - return in, tx, nil - } - - if parsedRequest.Body != nil && len(parsedRequest.Body) != 0 { - it, _, err := tx.WriteRequestBody(parsedRequest.Body) - if err != nil { - return nil, nil, errors.Wrap(err, "Cannot read request body") - } - - if it != nil { - //log.Infof("blocking rule id %d", in.RuleID) - return it, nil, nil - } - } - - in, err := tx.ProcessRequestBody() - if err != nil { - return nil, nil, errors.Wrap(err, "Cannot process request body") - } - if in != nil && wafType == InBand { - //log.Infof("blocking rule id %d", in.RuleID) - - return in, tx, nil - } - - return nil, tx, nil -} - -func (r *WafRunner) Run(t *tomb.Tomb) error { - r.logger.Infof("Waf Runner ready to process event") - for { - select { - case <-t.Dying(): - r.logger.Infof("Waf Runner is dying") - return nil - case request := <-r.inChan: - r.logger.Infof("Requests handled by runner %s", r.UUID) - var evt *types.Event - WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() - //measure the time spent in the WAF - startParsing := time.Now() - inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID) - expTx := inBoundTx.(experimental.FullTransaction) - // we use this internal transaction for the expr helpers - tx := waf.NewTransaction(expTx) - - //r.logger.Infof("Processing request %s | tx: %p", request.UUID, tx) - - //Run the pre_eval hooks - for _, rules := range r.RulesCollections { - if len(rules.CompiledPreEval) == 0 { - continue - } - for _, compiledHook := range rules.CompiledPreEval { - if compiledHook.Filter != nil { - res, err := expr.Run(compiledHook.Filter, map[string]interface{}{ - "rules": rules, - "req": request, - }) - if err != nil { - log.Errorf("unable to run PreEval filter: %s", err) - continue - } - - switch t := res.(type) { - case bool: - if !t { - log.Infof("filter didnt match") - continue - } - default: - log.Errorf("Filter must return a boolean, can't filter") - continue - } - } - // here means there is no filter or the filter matched - for _, applyExpr := range compiledHook.Apply { - _, err := expr.Run(applyExpr, map[string]interface{}{ - "rules": rules, - "req": request, - "RemoveRuleByID": tx.RemoveRuleByIDWithError, - }) - if err != nil { - log.Errorf("unable to apply filter: %s", err) - continue - } - } - } - } - - in, expTx, err := r.processReqWithEngine(expTx, request, InBand) - request.Tx = expTx - //log.Infof("-> %s", spew.Sdump(in)) - - //log.Infof("tx variables: %+v", expTx.Collection(variables.TX)) - - //foo := expTx.(plugintypes.TransactionState) - - //log.Infof("from tstate: %+v", foo.Variables().TX().FindAll()) - - response := waf.NewResponseRequest(expTx, in, request.UUID, err) - - // run the on_match hooks - for _, rules := range r.RulesCollections { - if len(rules.CompiledOnMatch) == 0 { - continue - } - for _, compiledHook := range rules.CompiledOnMatch { - if compiledHook.Filter != nil { - res, err := expr.Run(compiledHook.Filter, map[string]interface{}{ - "rules": rules, - "req": request, - }) - if err != nil { - r.logger.Errorf("unable to run PreEval filter: %s", err) - continue - } - - switch t := res.(type) { - case bool: - if !t { - continue - } - default: - r.logger.Errorf("Filter must return a boolean, can't filter") - continue - } - } - // here means there is no filter or the filter matched - for _, applyExpr := range compiledHook.Apply { - _, err := expr.Run(applyExpr, map[string]interface{}{ - "rules": rules, - "req": request, - "RemoveRuleByID": tx.RemoveRuleByIDWithError, - "SetRemediation": response.SetRemediation, - "SetRemediationByID": response.SetRemediationByID, - "CancelEvent": response.CancelEvent, - }) - if err != nil { - r.logger.Errorf("unable to apply filter: %s", err) - continue - } - } - } - } - logged := false - //measure the full time spent in the WAF - elapsed := time.Since(startParsing) - WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) - // send back the result to the HTTP handler for the InBand part - request.ResponseChannel <- response - if in != nil && response.SendEvents { - evt = &types.Event{} - *evt, err = EventFromRequest(request) - if err != nil { - return fmt.Errorf("cannot create event from waap context : %w", err) - } - err = r.AccumulateTxToEvent(expTx, InBand, evt) - if err != nil { - return fmt.Errorf("cannot convert transaction to event : %w", err) - } - LogWaapEvent(evt, r.logger) - logged = true - r.outChan <- *evt - } - expTx.Close() - - outBandStart := time.Now() - // Process outBand - outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) - expTx = outBandTx.(experimental.FullTransaction) - - in, expTx, err = r.processReqWithEngine(expTx, request, OutOfBand) - if err != nil { //things went south - r.logger.Errorf("Error while processing request : %s", err) - continue - } - request.Tx = expTx - if expTx != nil && len(expTx.MatchedRules()) > 0 { - //if event was not instantiated after inband processing, do it now - if evt == nil { - tmpEvt, err := EventFromRequest(request) - if err != nil { - return fmt.Errorf("cannot create event from waap context : %w", err) - } - evt = &tmpEvt - } - - err = r.AccumulateTxToEvent(expTx, OutOfBand, evt) - if err != nil { - return fmt.Errorf("cannot convert transaction to event : %w", err) - } - - // expTx.MatchedRules() returns also rules that set variables - // in evt.Waap.MatchedRules we have filtered those rules - if len(evt.Waap.MatchedRules) > 0 { - if !logged { - LogWaapEvent(evt, r.logger) - } - r.outChan <- *evt - } - } - expTx.Close() - //measure the full time spent in the WAF - totalElapsed := time.Since(startParsing) - WafGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(totalElapsed.Seconds()) - elapsed = time.Since(outBandStart) - WafOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) - } - } -} - -type BodyResponse struct { - Action string `json:"action"` -} - -func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { - // parse the request only once - parsedRequest, err := waf.NewParsedRequestFromRequest(r) - if err != nil { - log.Errorf("%s", err) - rw.WriteHeader(http.StatusInternalServerError) - return - } - w.InChan <- parsedRequest - - message := <-parsedRequest.ResponseChannel - - if message.Err != nil { - log.Errorf("Error while processing InBAND: %s", err) - rw.WriteHeader(http.StatusInternalServerError) - return - } - - if message.Interruption != nil { - rw.WriteHeader(http.StatusForbidden) - action := message.Interruption.Action - if action == "deny" { // bouncers understand "ban" and not "deny" - action = "ban" - } - body, err := json.Marshal(BodyResponse{Action: action}) - if err != nil { - log.Errorf("unable to build response: %s", err) - } else { - rw.Write(body) - } - return - } - - rw.WriteHeader(http.StatusOK) - body, err := json.Marshal(BodyResponse{Action: "allow"}) - if err != nil { - log.Errorf("unable to marshal response: %s", err) - rw.WriteHeader(http.StatusInternalServerError) - } else { - rw.Write(body) - } - -} diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 3a5de2be501..196a5532a69 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -1,16 +1,26 @@ package waf -import "github.com/crowdsecurity/coraza/v3/experimental" +import ( + "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/experimental" +) -type Transaction struct { +type ExtendedTransaction struct { Tx experimental.FullTransaction } -func NewTransaction(tx experimental.FullTransaction) Transaction { - return Transaction{Tx: tx} +func NewExtendedTransaction(engine coraza.WAF, uuid string) ExtendedTransaction { + inBoundTx := engine.NewTransactionWithID(uuid) + expTx := inBoundTx.(experimental.FullTransaction) + tx := NewTransaction(expTx) + return tx } -func (t *Transaction) RemoveRuleByIDWithError(id int) error { +func NewTransaction(tx experimental.FullTransaction) ExtendedTransaction { + return ExtendedTransaction{Tx: tx} +} + +func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { t.Tx.RemoveRuleByID(id) return nil } @@ -18,8 +28,8 @@ func (t *Transaction) RemoveRuleByIDWithError(id int) error { func GetEnv() map[string]interface{} { ResponseRequest := ResponseRequest{} ParsedRequest := ParsedRequest{} - Rules := &WafRulesCollection{} - Tx := Transaction{} + Rules := &WaapCollection{} + Tx := ExtendedTransaction{} return map[string]interface{}{ "rules": Rules, diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go new file mode 100644 index 00000000000..511b5c1180b --- /dev/null +++ b/pkg/waf/waap.go @@ -0,0 +1,237 @@ +package waf + +import ( + "fmt" + "regexp" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + corazatypes "github.com/crowdsecurity/coraza/v3/types" + log "github.com/sirupsen/logrus" +) + +type Hook struct { + Filter string `yaml:"filter"` + FilterExpr *vm.Program `yaml:"-"` + + OnSuccess string `yaml:"on_success"` + Apply []string `yaml:"apply"` + ApplyExpr []*vm.Program `yaml:"-"` +} + +func (h *Hook) Build() error { + + if h.Filter != "" { + program, err := expr.Compile(h.Filter) //FIXME: opts + if err != nil { + return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err) + } + h.FilterExpr = program + } + for _, apply := range h.Apply { + program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...) + if err != nil { + return fmt.Errorf("unable to compile apply %s : %w", apply, err) + } + h.ApplyExpr = append(h.ApplyExpr, program) + } + return nil +} + +// runtime version of WaapConfig +type WaapRuntimeConfig struct { + Name string + OutOfBandRules []WaapCollection + OutOfBandTx ExtendedTransaction //is it a good idea ? + InBandRules []WaapCollection + InBandTx ExtendedTransaction //is it a good idea ? + DefaultRemediation string + CompiledOnLoad []Hook + CompiledPreEval []Hook + CompiledOnMatch []Hook + CompiledVariablesTracking []*regexp.Regexp +} + +type WaapConfig struct { + Name string `yaml:"name"` + OutOfBandRules []string `yaml:"outofband_rules"` + InBandRules []string `yaml:"inband_rules"` + DefaultRemediation string `yaml:"default_remediation"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + OnMatch []Hook `yaml:"on_match"` + VariablesTracking []string `yaml:"variables_tracking"` +} + +func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { + ret := &WaapRuntimeConfig{} + ret.Name = wc.Name + ret.DefaultRemediation = wc.DefaultRemediation + + //load rules + for _, rule := range wc.OutOfBandRules { + collection, err := LoadCollection(rule) + if err != nil { + return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) + } + ret.OutOfBandRules = append(ret.OutOfBandRules, collection) + } + + for _, rule := range wc.InBandRules { + collection, err := LoadCollection(rule) + if err != nil { + return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) + } + ret.InBandRules = append(ret.InBandRules, collection) + } + + //load hooks + for _, hook := range wc.OnLoad { + err := hook.Build() + if err != nil { + return nil, fmt.Errorf("unable to build on_load hook : %s", err) + } + ret.CompiledOnLoad = append(ret.CompiledOnLoad, hook) + } + + for _, hook := range wc.PreEval { + err := hook.Build() + if err != nil { + return nil, fmt.Errorf("unable to build pre_eval hook : %s", err) + } + ret.CompiledPreEval = append(ret.CompiledPreEval, hook) + } + + for _, hook := range wc.OnMatch { + err := hook.Build() + if err != nil { + return nil, fmt.Errorf("unable to build on_match hook : %s", err) + } + ret.CompiledOnMatch = append(ret.CompiledOnMatch, hook) + } + + //variable tracking + for _, variable := range wc.VariablesTracking { + compiledVariableRule, err := regexp.Compile(variable) + if err != nil { + return nil, fmt.Errorf("cannot compile variable regexp %s: %w", variable, err) + } + ret.CompiledVariablesTracking = append(ret.CompiledVariablesTracking, compiledVariableRule) + } + return ret, nil +} + +func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest, response ResponseRequest) error { + + for _, rule := range w.CompiledOnMatch { + if rule.FilterExpr != nil { + output, err := expr.Run(rule.FilterExpr, map[string]interface{}{ + //"rules": rules, //is it still useful ? + "req": request, + }) + if err != nil { + return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Infof("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + for _, applyExpr := range rule.ApplyExpr { + _, err := expr.Run(applyExpr, map[string]interface{}{ + //"rules": w.InBandTx.Tx.Rules, //what is it supposed to be ? matched rules ? + "req": request, + "RemoveInbandRuleByID": w.RemoveInbandRuleByID, + "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, + "SetRemediation": response.SetRemediation, + "SetRemediationByID": response.SetRemediationByID, + "CancelEvent": response.CancelEvent, + }) + if err != nil { + log.Errorf("unable to apply filter: %s", err) + continue + } + } + } + return nil +} + +func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { + for _, rule := range w.CompiledPreEval { + if rule.FilterExpr != nil { + output, err := expr.Run(rule.FilterExpr, map[string]interface{}{ + //"rules": rules, //is it still useful ? + "req": request, + }) + if err != nil { + return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Infof("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range rule.ApplyExpr { + _, err := expr.Run(applyExpr, map[string]interface{}{ + "inband_rules": w.InBandRules, + "outband_rules": w.OutOfBandRules, + "req": request, + "RemoveInbandRuleByID": w.RemoveInbandRuleByID, + "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, + }) + if err != nil { + log.Errorf("unable to apply filter: %s", err) + continue + } + } + } + + return nil +} + +func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) { + w.InBandTx.RemoveRuleByIDWithError(id) +} + +func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) { + w.OutOfBandTx.RemoveRuleByIDWithError(id) +} + +func (w *WaapRuntimeConfig) ProcessInBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { + for _, rule := range w.InBandRules { + interrupt, err := rule.Eval(request) + if err != nil { + return nil, fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) + } + if interrupt != nil { + return interrupt, nil + } + } + return nil, nil +} + +func (w *WaapRuntimeConfig) ProcessOutOfBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { + for _, rule := range w.OutOfBandRules { + interrupt, err := rule.Eval(request) + if err != nil { + return nil, fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) + } + if interrupt != nil { + return interrupt, nil + } + } + return nil, nil +} diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go new file mode 100644 index 00000000000..ce7e605eced --- /dev/null +++ b/pkg/waf/waap_rules_collection.go @@ -0,0 +1,34 @@ +package waf + +import corazatypes "github.com/crowdsecurity/coraza/v3/types" + +// to be filled w/ seb update +type WaapCollection struct { +} + +// to be filled w/ seb update +type WaapCollectionConfig struct { + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + MergedRules []string `yaml:"-"` +} + +func LoadCollection(collection string) (WaapCollection, error) { + return WaapCollection{}, nil +} + +func (wcc WaapCollectionConfig) LoadCollection(collection string) (WaapCollection, error) { + return WaapCollection{}, nil +} + +func (w WaapCollection) Check() error { + return nil +} + +func (w WaapCollection) Eval(req ParsedRequest) (*corazatypes.Interruption, error) { + return nil, nil +} + +func (w WaapCollection) GetDisplayName() string { + return "rule XX" +} diff --git a/pkg/waf/waf.go b/pkg/waf/waf.go deleted file mode 100644 index c1755bff36b..00000000000 --- a/pkg/waf/waf.go +++ /dev/null @@ -1,204 +0,0 @@ -package waf - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" -) - -type Hook struct { - Filter string `yaml:"filter"` - FilterExpr *vm.Program `yaml:"-"` - OnSuccess string `yaml:"on_success"` - Apply []string `yaml:"apply"` - ApplyExpr []*vm.Program `yaml:"-"` -} - -type CompiledHook struct { - Filter *vm.Program `yaml:"-"` - Apply []*vm.Program `yaml:"-"` -} - -/*type WafConfig struct { - InbandRules []WafRule - OutOfBandRules []WafRule - Datadir string - logger *log.Entry -}*/ - -// This represents one "waf-rule" config -type WafConfig struct { - SecLangFilesRules []string `yaml:"seclang_files_rules"` - SecLangRules []string `yaml:"seclang_rules"` - OnLoad []Hook `yaml:"on_load"` - PreEval []Hook `yaml:"pre_eval"` - OnMatch []Hook `yaml:"on_match"` - - CompiledOnLoad []CompiledHook `yaml:"-"` - CompiledPreEval []CompiledHook `yaml:"-"` - CompiledOnMatch []CompiledHook `yaml:"-"` - - MergedRules []string `yaml:"-"` - OutOfBand bool `yaml:"outofband"` -} - -type WafRuleLoader struct { - logger *log.Entry - Datadir string -} - -func buildHook(hook Hook) (CompiledHook, error) { - compiledHook := CompiledHook{} - if hook.Filter != "" { - program, err := expr.Compile(hook.Filter) //FIXME: opts - if err != nil { - return CompiledHook{}, fmt.Errorf("unable to compile filter %s : %w", hook.Filter, err) - } - compiledHook.Filter = program - } - for _, apply := range hook.Apply { - program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...) - if err != nil { - return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err) - } - compiledHook.Apply = append(compiledHook.Apply, program) - } - return compiledHook, nil -} - -func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) { - var wafRulesFiles []string - for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { - if hubWafRuleItem.Installed { - wafRulesFiles = append(wafRulesFiles, hubWafRuleItem.LocalPath) - } - } - - if len(wafRulesFiles) == 0 { - return nil, fmt.Errorf("no waf rules found in hub") - } - - w.logger.Infof("Loading %d waf files", len(wafRulesFiles)) - wafRulesCollections := []*WafRulesCollection{} - for _, wafRulesFile := range wafRulesFiles { - - fileContent, err := os.ReadFile(wafRulesFile) - if err != nil { - w.logger.Errorf("unable to read file %s : %s", wafRulesFile, err) - continue - } - wafConfig := WafConfig{} - err = yaml.Unmarshal(fileContent, &wafConfig) - if err != nil { - w.logger.Errorf("unable to unmarshal file %s : %s", wafRulesFile, err) - continue - } - - //spew.Dump(wafConfig) - - collection := &WafRulesCollection{} - - if wafConfig.SecLangFilesRules != nil { - for _, rulesFile := range wafConfig.SecLangFilesRules { - fullPath := filepath.Join(w.Datadir, rulesFile) - c, err := os.ReadFile(fullPath) - if err != nil { - w.logger.Errorf("unable to read file %s : %s", rulesFile, err) - continue - } - for _, line := range strings.Split(string(c), "\n") { - if strings.HasPrefix(line, "#") { - continue - } - if strings.TrimSpace(line) == "" { - continue - } - collection.Rules = append(collection.Rules, WafRule{RawRule: line}) - } - } - } - - if wafConfig.SecLangRules != nil { - for _, rule := range wafConfig.SecLangRules { - collection.Rules = append(collection.Rules, WafRule{RawRule: rule}) - } - } - - //TODO: add our own format - - //compile hooks - for _, hook := range wafConfig.OnLoad { - compiledHook, err := buildHook(hook) - if err != nil { - w.logger.Errorf("unable to build on_load hook %s : %s", hook.Filter, err) - continue - } - collection.CompiledOnLoad = append(collection.CompiledOnLoad, compiledHook) - } - - for _, hook := range wafConfig.PreEval { - compiledHook, err := buildHook(hook) - if err != nil { - w.logger.Errorf("unable to build pre_eval hook %s : %s", hook.Filter, err) - continue - } - collection.CompiledPreEval = append(collection.CompiledPreEval, compiledHook) - } - - for _, hook := range wafConfig.OnMatch { - compiledHook, err := buildHook(hook) - if err != nil { - w.logger.Errorf("unable to build on_match hook %s : %s", hook.Filter, err) - continue - } - collection.CompiledOnMatch = append(collection.CompiledOnMatch, compiledHook) - } - - //Run the on_load hooks - if len(collection.CompiledOnLoad) > 0 { - w.logger.Infof("Running %d on_load hooks", len(collection.CompiledOnLoad)) - for hookIdx, onLoadHook := range collection.CompiledOnLoad { - //Ignore filter for on load ? - if onLoadHook.Apply != nil { - for exprIdx, applyExpr := range onLoadHook.Apply { - _, err := expr.Run(applyExpr, map[string]interface{}{ - "rules": collection, - }) - if err != nil { - w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err) - continue - } - } - } - } - } - wafRulesCollections = append(wafRulesCollections, collection) - } - - return wafRulesCollections, nil -} - -func NewWafRuleLoader() *WafRuleLoader { - //FIXME: find a better way to get the datadir - clog := log.New() - if err := types.ConfigureLogger(clog); err != nil { - //return nil, fmt.Errorf("while configuring datasource logger: %w", err) - return nil - } - logger := clog.WithFields(log.Fields{ - "type": "waf-config", - }) - - initWafHelpers() - - return &WafRuleLoader{Datadir: csconfig.DataDir, logger: logger} -} diff --git a/pkg/waf/waf_rules_collection.go b/pkg/waf/waf_rules_collection.go deleted file mode 100644 index f6669527f9e..00000000000 --- a/pkg/waf/waf_rules_collection.go +++ /dev/null @@ -1,37 +0,0 @@ -package waf - -import "strings" - -type WafRule struct { - RawRule string - RuleID string - InBand bool -} - -// This is the "compiled" state of a WafConfig -type WafRulesCollection struct { - Rules []WafRule - CompiledOnLoad []CompiledHook `yaml:"-"` - CompiledPreEval []CompiledHook `yaml:"-"` - CompiledOnMatch []CompiledHook `yaml:"-"` - OutOfBand bool -} - -func (w *WafRulesCollection) SetInBand() error { - w.OutOfBand = false - return nil -} - -func (w *WafRulesCollection) SetOutOfBand() error { - w.OutOfBand = true - return nil -} - -func (w *WafRulesCollection) String() string { - //return strings.Join(w.Rules, "\n") - var rules []string - for _, rule := range w.Rules { - rules = append(rules, rule.RawRule) - } - return strings.Join(rules, "\n") -} From 1a5799e0582e176a01ccccb209fa6a18f1f3b19f Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 12 Sep 2023 09:45:14 +0200 Subject: [PATCH 072/263] up --- pkg/acquisition/modules/{waf => waap}/README.md | 0 pkg/acquisition/modules/{waf => waap}/metrics.go | 0 pkg/acquisition/modules/{waf => waap}/rx_operator.go | 0 pkg/acquisition/modules/{waf => waap}/utils.go | 0 pkg/acquisition/modules/{waf => waap}/waap.go | 0 pkg/acquisition/modules/{waf => waap}/waap_runner.go | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename pkg/acquisition/modules/{waf => waap}/README.md (100%) rename pkg/acquisition/modules/{waf => waap}/metrics.go (100%) rename pkg/acquisition/modules/{waf => waap}/rx_operator.go (100%) rename pkg/acquisition/modules/{waf => waap}/utils.go (100%) rename pkg/acquisition/modules/{waf => waap}/waap.go (100%) rename pkg/acquisition/modules/{waf => waap}/waap_runner.go (100%) diff --git a/pkg/acquisition/modules/waf/README.md b/pkg/acquisition/modules/waap/README.md similarity index 100% rename from pkg/acquisition/modules/waf/README.md rename to pkg/acquisition/modules/waap/README.md diff --git a/pkg/acquisition/modules/waf/metrics.go b/pkg/acquisition/modules/waap/metrics.go similarity index 100% rename from pkg/acquisition/modules/waf/metrics.go rename to pkg/acquisition/modules/waap/metrics.go diff --git a/pkg/acquisition/modules/waf/rx_operator.go b/pkg/acquisition/modules/waap/rx_operator.go similarity index 100% rename from pkg/acquisition/modules/waf/rx_operator.go rename to pkg/acquisition/modules/waap/rx_operator.go diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waap/utils.go similarity index 100% rename from pkg/acquisition/modules/waf/utils.go rename to pkg/acquisition/modules/waap/utils.go diff --git a/pkg/acquisition/modules/waf/waap.go b/pkg/acquisition/modules/waap/waap.go similarity index 100% rename from pkg/acquisition/modules/waf/waap.go rename to pkg/acquisition/modules/waap/waap.go diff --git a/pkg/acquisition/modules/waf/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go similarity index 100% rename from pkg/acquisition/modules/waf/waap_runner.go rename to pkg/acquisition/modules/waap/waap_runner.go From 5a0b1b72d350d4c4b94adf989ec1f14981720e9a Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 12 Sep 2023 10:42:28 +0200 Subject: [PATCH 073/263] up --- cmd/crowdsec/metrics.go | 6 +++--- pkg/acquisition/acquisition.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 5b89eec3d37..2c1b2970b62 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -12,7 +12,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/go-cs-lib/version" - waf "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" + waap "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waap" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -171,7 +171,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, leaky.BucketsCurrentCount, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - waf.WafGlobalParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + waap.WafGlobalParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, ) } else { log.Infof("Loading prometheus collectors") @@ -182,7 +182,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - waf.WafGlobalParsingHistogram, waf.WafInbandParsingHistogram, waf.WafOutbandParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + waap.WafGlobalParsingHistogram, waap.WafInbandParsingHistogram, waap.WafOutbandParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, ) } diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index b37f9f985d4..264f4814579 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -27,7 +27,7 @@ import ( k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit" s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" - wafacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" + wafacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waap" wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" From 1286efc74f588180ce7bdfd423a598026c45bbd5 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 12 Sep 2023 18:17:58 +0200 Subject: [PATCH 074/263] up --- pkg/acquisition/modules/waap/README.md | 112 +++++++------------------ pkg/acquisition/modules/waap/waap.go | 19 +++++ pkg/waf/waap.go | 15 ++++ 3 files changed, 64 insertions(+), 82 deletions(-) diff --git a/pkg/acquisition/modules/waap/README.md b/pkg/acquisition/modules/waap/README.md index 459672f5f02..9899b137257 100644 --- a/pkg/acquisition/modules/waap/README.md +++ b/pkg/acquisition/modules/waap/README.md @@ -1,31 +1,12 @@ -Ongoing poc for Coraza +Ongoing poc for Coraza WAAP -For config: +# Configuration pieces -coraza_inband.conf: -```shell -SecRuleEngine On -SecRule ARGS:id "@eq 0" "id:1, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" -SecRequestBodyAccess On -SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" -``` - - -coraza_outofband.conf: -```shell -SecRuleEngine On -SecRule ARGS:id "@eq 1" "id:3,phase:1,log,msg:'Invalid id',log,auditlog" -SecRule ARGS:idd "@eq 2" "id:4,phase:1,log,msg:'Invalid id',log,auditlog" -SecRequestBodyAccess On -#We know that because we are not cloning the body in waf.go, the outofband rules cannot access body as it has been consumed. -#We are finding a way around this -#SecRule REQUEST_BODY "@contains totolol" "id:4, phase:2,deny,msg:'Invalid request body',log,auditlog" -#SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" - -``` +## Acquisition +acquisition example: -acquis.yaml : +> `config/acquis.yaml` : ```yaml listen_addr: 127.0.0.1 @@ -34,71 +15,38 @@ path: / source: waf labels: type: waf +#routines: 1 +waap_config: mytest ``` -Coraza parser: +## Waap config -```yaml -onsuccess: next_stage -debug: true -filter: "evt.Parsed.program == 'waf'" -name: crowdsecurity/waf-logs -description: "Parse WAF logs" -statics: - - parsed: cloudtrail_parsed - expression: UnmarshalJSON(evt.Line.Raw, evt.Unmarshaled, 'waf') - - meta: req_uuid - expression: evt.Unmarshaled.waf.req_uuid - - meta: source_ip - expression: evt.Unmarshaled.waf.source_ip - - meta: rule_id - expression: evt.Unmarshaled.waf.rule_id - - meta: action - expression: evt.Unmarshaled.waf.rule_action - - meta: service - value: waf - - parsed: event_type - value: waf_match - -``` +The waap config defines what rules that will be loaded by a given waap engine (associated with an acquis). -Coraza trigger scenario: +> `config/waap_configs/mytest.yaml` ```yaml -type: trigger -filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "inband" -debug: true -name: coroza-triggger -description: here we go -blackhole: 2m -labels: - type: exploit - remediation: true -groupby: "evt.Meta.source_ip" +name: mytest.yaml +outofband_rules: + - crowdsec/crs-default +inband_rules: + - crowdsec/vpatch-default +default_remediation: block +variables_tracking: + - session_* +# onload: +# - apply: +# - DisabledInBandRuleByID(1003) +# pre_eval: +# - filter: evt.SourceIP == '1.3.4.5' +# apply: +# - DisableOutOfBandRuleByID(2302) ``` -Coraza leaky scenario: - -```yaml -type: leaky -filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "outofband" -debug: true -name: coroza-leaky -description: here we go -blackhole: 2m -leakspeed: 30s -capacity: 1 -labels: - type: exploit - remediation: true -groupby: "evt.Meta.source_ip" -distinct: evt.Meta.rule_id -``` - - - -To be solved: - - We need to solve the body cloning issue - - Merge w/ hub +# Waap Rules +For the above two to work, we need to have the two refered waap collection installed : `crowdsec/crs-default` and `crowdsec/vpatch-default`. You need to set hub_branch to ... +```yaml +cscli waf-rules install ... +``` \ No newline at end of file diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index f34c1bd86fd..2485a82e53e 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -34,6 +34,7 @@ type WaapSourceConfig struct { Routines int `yaml:"routines"` Debug bool `yaml:"debug"` WaapConfig string `yaml:"waap_config"` + WaapConfigPath string `yaml:"waap_config_path"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -118,6 +119,24 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.InChan = make(chan waf.ParsedRequest) + + //let's load the associated waap_config: + if wc.WaapConfigPath != "" { + return fmt.Errorf("resolution gor waap_config not implemented yet") + } else if wc.WaapConfig != "" { + waapCfg := waf.WaapConfig{} + err := waapCfg.Load(wc.WaapConfig) + if err != nil { + return fmt.Errorf("unable to load waap_config : %s", err) + } + w.WaapRuntime, err = waapCfg.Build() + if err != nil { + return fmt.Errorf("unable to build waap_config : %s", err) + } + } else { + return fmt.Errorf("no waap_config provided") + } + w.WaapRunners = make([]WaapRunner, wc.Routines) for nbRoutine := 0; nbRoutine < wc.Routines; nbRoutine++ { diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 511b5c1180b..e0318d8abb1 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -2,12 +2,14 @@ package waf import ( "fmt" + "os" "regexp" "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" corazatypes "github.com/crowdsecurity/coraza/v3/types" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type Hook struct { @@ -63,6 +65,19 @@ type WaapConfig struct { VariablesTracking []string `yaml:"variables_tracking"` } +func (wc *WaapConfig) Load(file string) error { + yamlFile, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("unable to read file %s : %s", file, err) + } + err = yaml.UnmarshalStrict(yamlFile, wc) + if err != nil { + return fmt.Errorf("unable to parse yaml file %s : %s", file, err) + } + return nil + +} + func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret := &WaapRuntimeConfig{} ret.Name = wc.Name From 6930b1e3e5ae8d1cdfcdd963769befb68c2ca9ea Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 13 Sep 2023 10:45:06 +0200 Subject: [PATCH 075/263] up --- pkg/acquisition/modules/waap/waap.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 2485a82e53e..68200b32a19 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -53,7 +53,8 @@ type WaapSource struct { } func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { - err := yaml.UnmarshalStrict(yamlConfig, wc.config) + + err := yaml.UnmarshalStrict(yamlConfig, &wc.config) if err != nil { return errors.Wrap(err, "Cannot parse waf configuration") } @@ -82,6 +83,10 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { if wc.config.Routines == 0 { wc.config.Routines = 1 } + + if wc.config.WaapConfig == "" && wc.config.WaapConfigPath == "" { + return fmt.Errorf("waap_config or waap_config_path must be set") + } return nil } @@ -99,17 +104,17 @@ func logError(error corazatypes.MatchedRule) { } func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { - wc := WaapSourceConfig{} + //wc := WaapSourceConfig{} err := w.UnmarshalConfig(yamlConfig) if err != nil { return errors.Wrap(err, "unable to parse waf configuration") } w.logger = logger - w.config = wc + //w.config = wc w.logger.Tracef("WAF configuration: %+v", w.config) - w.addr = fmt.Sprintf("%s:%d", wc.ListenAddr, wc.ListenPort) + w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) w.mux = http.NewServeMux() @@ -121,11 +126,9 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.InChan = make(chan waf.ParsedRequest) //let's load the associated waap_config: - if wc.WaapConfigPath != "" { - return fmt.Errorf("resolution gor waap_config not implemented yet") - } else if wc.WaapConfig != "" { + if w.config.WaapConfigPath != "" { waapCfg := waf.WaapConfig{} - err := waapCfg.Load(wc.WaapConfig) + err := waapCfg.Load(w.config.WaapConfigPath) if err != nil { return fmt.Errorf("unable to load waap_config : %s", err) } @@ -133,19 +136,21 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { if err != nil { return fmt.Errorf("unable to build waap_config : %s", err) } + } else if w.config.WaapConfig != "" { + return fmt.Errorf("resolution of waap_config not implemented yet") } else { return fmt.Errorf("no waap_config provided") } - w.WaapRunners = make([]WaapRunner, wc.Routines) + w.WaapRunners = make([]WaapRunner, w.config.Routines) - for nbRoutine := 0; nbRoutine < wc.Routines; nbRoutine++ { + for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { wafUUID := uuid.New().String() wafLogger := &log.Entry{} //configure logger - if wc.Debug { + if w.config.Debug { var clog = log.New() if err := types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) From c435447d8ea2475547aefc723e7802e0971625ae Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 13 Sep 2023 10:57:29 +0200 Subject: [PATCH 076/263] up --- pkg/waf/waap.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index e0318d8abb1..325da850f06 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -217,6 +217,13 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { return nil } +/* @sbl / @tko +add the helpers to: + - remove by id-range + - remove by tag + +*/ + func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) { w.InBandTx.RemoveRuleByIDWithError(id) } From 2e60e8021c7b38f0e028fa5ffe8652e482b59939 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 13 Sep 2023 17:12:09 +0200 Subject: [PATCH 077/263] up wip --- pkg/acquisition/modules/waap/waap.go | 19 +-- pkg/acquisition/modules/waap/waap_runner.go | 5 +- pkg/waf/env.go | 23 ++-- pkg/waf/request.go | 43 +++---- pkg/waf/waap.go | 122 ++++++++++++++++---- pkg/waf/waf_helpers.go | 9 -- 6 files changed, 144 insertions(+), 77 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 68200b32a19..43d6fcc8aa6 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -52,6 +52,11 @@ type WaapSource struct { WaapRunners []WaapRunner //one for each go-routine } +// @tko + @sbl : we might want to get rid of that or improve it +type BodyResponse struct { + Action string `json:"action"` +} + func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &wc.config) @@ -165,10 +170,13 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { }) } + //we copy WaapRutime for each runner + wrt := *w.WaapRuntime runner := WaapRunner{ - inChan: w.InChan, - UUID: wafUUID, - logger: wafLogger, + inChan: w.InChan, + UUID: wafUUID, + logger: wafLogger, + WaapRuntime: &wrt, } w.WaapRunners[nbRoutine] = runner //most likely missign somethign here to actually start the runner :) @@ -247,10 +255,6 @@ func (w *WaapSource) Dump() interface{} { return w } -type BodyResponse struct { - Action string `json:"action"` -} - // should this be in the runner ? func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { // parse the request only once @@ -264,6 +268,7 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { message := <-parsedRequest.ResponseChannel + //@tko this parts needs to be redone if message.Err != nil { log.Errorf("Error while processing InBAND: %s", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 979a958b9d3..06c5a78a765 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -31,10 +31,10 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { return nil case request := <-r.inChan: r.logger.Infof("Requests handled by runner %s", request.UUID) + r.WaapRuntime.ClearResponse() - //tx := waf.NewExtendedTransaction(r.WaapInbandEngine, r.UUID) WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() - //measure the time spent in the WAF + //to measure the time spent in the WAF startParsing := time.Now() //pre eval (expr) rules @@ -43,7 +43,6 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Errorf("unable to process PreEval rules: %s", err) continue } - //inband WAAP rules interrupt, err := r.WaapRuntime.ProcessInBandRules(request) elapsed := time.Since(startParsing) diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 196a5532a69..7e5f78c3546 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -25,18 +25,17 @@ func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { return nil } -func GetEnv() map[string]interface{} { - ResponseRequest := ResponseRequest{} - ParsedRequest := ParsedRequest{} - Rules := &WaapCollection{} - Tx := ExtendedTransaction{} - +// simply used to ease the compilation & runtime of the hooks +func GetHookEnv(w WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { return map[string]interface{}{ - "rules": Rules, - "req": ParsedRequest, - "SetRemediation": ResponseRequest.SetRemediation, - "SetRemediationByID": ResponseRequest.SetRemediationByID, - "CancelEvent": ResponseRequest.CancelEvent, - "RemoveRuleByID": Tx.RemoveRuleByIDWithError, + "inband_rules": w.InBandRules, + "outband_rules": w.OutOfBandRules, + "req": request, + "RemoveInbandRuleByID": w.RemoveInbandRuleByID, + "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, + "SetAction": w.SetAction, + "SetHTTPCode": w.SetHTTPCode, + "SetActionByID": w.SetActionnByID, + "CancelEvent": w.CancelEvent, } } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index bed7a6ef90a..1ba53cc79f3 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -36,29 +36,29 @@ func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interru } } -func (r *ResponseRequest) SetRemediation(remediation string) error { - if r.Interruption == nil { - return nil - } - r.Interruption.Action = remediation - return nil -} +// func (r *ResponseRequest) SetRemediation(remediation string) error { +// if r.Interruption == nil { +// return nil +// } +// r.Interruption.Action = remediation +// return nil +// } -func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { - if r.Interruption == nil { - return nil - } - if r.Interruption.RuleID == ID { - r.Interruption.Action = remediation - } - return nil -} +// func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { +// if r.Interruption == nil { +// return nil +// } +// if r.Interruption.RuleID == ID { +// r.Interruption.Action = remediation +// } +// return nil +// } -func (r *ResponseRequest) CancelEvent() error { - // true by default - r.SendEvents = false - return nil -} +// func (r *ResponseRequest) CancelEvent() error { +// // true by default +// r.SendEvents = false +// return nil +// } type ParsedRequest struct { RemoteAddr string @@ -77,6 +77,7 @@ type ParsedRequest struct { ResponseChannel chan ResponseRequest } +// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { var err error body := make([]byte, 0) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 325da850f06..526cbac8870 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -21,6 +21,7 @@ type Hook struct { ApplyExpr []*vm.Program `yaml:"-"` } +// @tko : todo - debug mode func (h *Hook) Build() error { if h.Filter != "" { @@ -31,7 +32,7 @@ func (h *Hook) Build() error { h.FilterExpr = program } for _, apply := range h.Apply { - program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...) + program, err := expr.Compile(apply, GetExprWAFOptions(GetHookEnv(WaapRuntimeConfig{}, ParsedRequest{}))...) if err != nil { return fmt.Errorf("unable to compile apply %s : %w", apply, err) } @@ -40,18 +41,30 @@ func (h *Hook) Build() error { return nil } +type WaapTempResponse struct { + InBandInterrupt bool + OutOfBandInterrupt bool + Action string //allow, deny, captcha, log + HTTPResponseCode int + SendEvent bool //do we send an internal event on rule match +} + // runtime version of WaapConfig type WaapRuntimeConfig struct { Name string OutOfBandRules []WaapCollection - OutOfBandTx ExtendedTransaction //is it a good idea ? InBandRules []WaapCollection - InBandTx ExtendedTransaction //is it a good idea ? DefaultRemediation string CompiledOnLoad []Hook CompiledPreEval []Hook CompiledOnMatch []Hook CompiledVariablesTracking []*regexp.Regexp + Config *WaapConfig + + //those are ephemeral, created/destroyed with every req + OutOfBandTx ExtendedTransaction //is it a good idea ? + InBandTx ExtendedTransaction //is it a good idea ? + Response WaapTempResponse } type WaapConfig struct { @@ -59,12 +72,24 @@ type WaapConfig struct { OutOfBandRules []string `yaml:"outofband_rules"` InBandRules []string `yaml:"inband_rules"` DefaultRemediation string `yaml:"default_remediation"` + DefaultPassAction string `yaml:"default_pass_action"` + BlockedHTTPCode int `yaml:"blocked_http_code"` + PassedHTTPCode int `yaml:"passed_http_code"` OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` VariablesTracking []string `yaml:"variables_tracking"` } +func (w *WaapRuntimeConfig) ClearResponse() { + log.Infof("#-> %p", w) + w.Response = WaapTempResponse{} + log.Infof("-> %p", w.Config) + w.Response.Action = w.Config.DefaultPassAction + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + w.Response.SendEvent = true +} + func (wc *WaapConfig) Load(file string) error { yamlFile, err := os.ReadFile(file) if err != nil { @@ -74,6 +99,21 @@ func (wc *WaapConfig) Load(file string) error { if err != nil { return fmt.Errorf("unable to parse yaml file %s : %s", file, err) } + if wc.Name == "" { + return fmt.Errorf("name cannot be empty") + } + if wc.DefaultRemediation == "" { + return fmt.Errorf("default_remediation cannot be empty") + } + if wc.BlockedHTTPCode == 0 { + wc.BlockedHTTPCode = 403 + } + if wc.PassedHTTPCode == 0 { + wc.PassedHTTPCode = 200 + } + if wc.DefaultPassAction == "" { + wc.DefaultPassAction = "allow" + } return nil } @@ -81,6 +121,7 @@ func (wc *WaapConfig) Load(file string) error { func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret := &WaapRuntimeConfig{} ret.Name = wc.Name + ret.Config = wc ret.DefaultRemediation = wc.DefaultRemediation //load rules @@ -160,13 +201,12 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest, response } for _, applyExpr := range rule.ApplyExpr { _, err := expr.Run(applyExpr, map[string]interface{}{ - //"rules": w.InBandTx.Tx.Rules, //what is it supposed to be ? matched rules ? - "req": request, - "RemoveInbandRuleByID": w.RemoveInbandRuleByID, - "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, - "SetRemediation": response.SetRemediation, - "SetRemediationByID": response.SetRemediationByID, - "CancelEvent": response.CancelEvent, + // "req": request, + // "RemoveInbandRuleByID": w.RemoveInbandRuleByID, + // "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, + // "SetAction": response.SetAction, + // "SetRemediationByID": response.SetRemediationByID, + // "CancelEvent": response.CancelEvent, }) if err != nil { log.Errorf("unable to apply filter: %s", err) @@ -180,10 +220,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest, response func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, map[string]interface{}{ - //"rules": rules, //is it still useful ? - "req": request, - }) + output, err := expr.Run(rule.FilterExpr, GetHookEnv(*w, request)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -200,13 +237,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, map[string]interface{}{ - "inband_rules": w.InBandRules, - "outband_rules": w.OutOfBandRules, - "req": request, - "RemoveInbandRuleByID": w.RemoveInbandRuleByID, - "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, - }) + _, err := expr.Run(applyExpr, GetHookEnv(*w, request)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue @@ -221,15 +252,37 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { add the helpers to: - remove by id-range - remove by tag + - set remediation by tag/id-range */ -func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) { - w.InBandTx.RemoveRuleByIDWithError(id) +func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { + return w.InBandTx.RemoveRuleByIDWithError(id) } -func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) { - w.OutOfBandTx.RemoveRuleByIDWithError(id) +func (w *WaapRuntimeConfig) CancelEvent() error { + w.Response.SendEvent = false + return nil +} + +func (w *WaapRuntimeConfig) SetActionnByID(id int, action string) error { + panic("not implemented") + return nil +} + +func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { + return w.OutOfBandTx.RemoveRuleByIDWithError(id) +} + +func (w *WaapRuntimeConfig) SetAction(action string) error { + w.Response.Action = action + return nil + +} + +func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { + w.Response.HTTPResponseCode = code + return nil } func (w *WaapRuntimeConfig) ProcessInBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { @@ -257,3 +310,22 @@ func (w *WaapRuntimeConfig) ProcessOutOfBandRules(request ParsedRequest) (*coraz } return nil, nil } + +type BodyResponse struct { + Action string `json:"action"` + HTTPStatus int `json:"http_status"` +} + +func (w *WaapRuntimeConfig) GenerateResponse(interrupted bool) (BodyResponse, error) { + resp := BodyResponse{} + //if there is no interrupt, we should allow with default code + if !interrupted { + resp.Action = w.Config.DefaultPassAction + resp.HTTPStatus = w.Config.PassedHTTPCode + return resp, nil + } + resp.Action = w.Config.DefaultRemediation + resp.HTTPStatus = w.Config.BlockedHTTPCode + + return resp, nil +} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index b99732ca3b5..97b1f30f0c3 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -30,12 +30,3 @@ func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { } return baseHelpers } - -func SetRulesToInband(params ...any) (any, error) { - - return nil, nil -} - -func SetRulesToOutOfBand(params ...any) (any, error) { - return nil, nil -} From 7081666199a0056c0f5ef3409955e4eabc7fd6b0 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 13 Sep 2023 17:34:53 +0200 Subject: [PATCH 078/263] up --- pkg/acquisition/modules/waap/waap.go | 54 ++++++++++----------- pkg/acquisition/modules/waap/waap_runner.go | 8 +-- pkg/waf/request.go | 37 +++++++------- pkg/waf/waap.go | 14 +++--- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 43d6fcc8aa6..213f9d67cdf 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -266,33 +266,33 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { } w.InChan <- parsedRequest - message := <-parsedRequest.ResponseChannel - - //@tko this parts needs to be redone - if message.Err != nil { - log.Errorf("Error while processing InBAND: %s", err) - rw.WriteHeader(http.StatusInternalServerError) - return - } - - //here we must rely on WaapRuntimeConfig to know what to do - if message.Interruption != nil { - rw.WriteHeader(http.StatusForbidden) - action := message.Interruption.Action - if action == "deny" { // bouncers understand "ban" and not "deny" - action = "ban" - } - body, err := json.Marshal(BodyResponse{Action: action}) - if err != nil { - log.Errorf("unable to build response: %s", err) - } else { - rw.Write(body) - } - return - } - - rw.WriteHeader(http.StatusOK) - body, err := json.Marshal(BodyResponse{Action: "allow"}) + response := <-parsedRequest.ResponseChannel + + // //@tko this parts needs to be redone + // if message.Err != nil { + // log.Errorf("Error while processing InBAND: %s", err) + // rw.WriteHeader(http.StatusInternalServerError) + // return + // } + + // //here we must rely on WaapRuntimeConfig to know what to do + // if message.Interruption != nil { + // rw.WriteHeader(http.StatusForbidden) + // action := message.Interruption.Action + // if action == "deny" { // bouncers understand "ban" and not "deny" + // action = "ban" + // } + // body, err := json.Marshal(BodyResponse{Action: action}) + // if err != nil { + // log.Errorf("unable to build response: %s", err) + // } else { + // rw.Write(body) + // } + // return + // } + + rw.WriteHeader(response.HTTPResponseCode) + body, err := json.Marshal(BodyResponse{Action: response.Action}) if err != nil { log.Errorf("unable to marshal response: %s", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 06c5a78a765..7d43c30e7a7 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -44,22 +44,22 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { continue } //inband WAAP rules - interrupt, err := r.WaapRuntime.ProcessInBandRules(request) + err = r.WaapRuntime.ProcessInBandRules(request) elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) //generate reponse for the remediation component, based on the WAAP config + inband rules evaluation //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation - response := waf.NewResponseRequest(r.WaapRuntime.InBandTx.Tx, interrupt, request.UUID, err) + //response := waf.NewResponseRequest(r.WaapRuntime.InBandTx.Tx, interrupt, request.UUID, err) - err = r.WaapRuntime.ProcessOnMatchRules(request, response) + err = r.WaapRuntime.ProcessOnMatchRules(request) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) continue } // send back the result to the HTTP handler for the InBand part - request.ResponseChannel <- response + request.ResponseChannel <- r.WaapRuntime.Response } } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 1ba53cc79f3..6fd95834ef2 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -7,7 +7,6 @@ import ( "net/url" "github.com/crowdsecurity/coraza/v3/experimental" - corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/google/uuid" ) @@ -18,23 +17,23 @@ const ( IPHeaderName = "X-Crowdsec-Waf-Ip" ) -type ResponseRequest struct { - UUID string - Tx corazatypes.Transaction - Interruption *corazatypes.Interruption - Err error - SendEvents bool -} +// type ResponseRequest struct { +// UUID string +// Tx corazatypes.Transaction +// Interruption *corazatypes.Interruption +// Err error +// SendEvents bool +// } -func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest { - return ResponseRequest{ - UUID: UUID, - Tx: Tx, - Interruption: in, - Err: err, - SendEvents: true, - } -} +// func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest { +// return ResponseRequest{ +// UUID: UUID, +// Tx: Tx, +// Interruption: in, +// Err: err, +// SendEvents: true, +// } +// } // func (r *ResponseRequest) SetRemediation(remediation string) error { // if r.Interruption == nil { @@ -74,7 +73,7 @@ type ParsedRequest struct { TransferEncoding []string UUID string Tx experimental.FullTransaction - ResponseChannel chan ResponseRequest + ResponseChannel chan WaapTempResponse } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine @@ -129,6 +128,6 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { Proto: r.Proto, Body: body, TransferEncoding: r.TransferEncoding, - ResponseChannel: make(chan ResponseRequest), + ResponseChannel: make(chan WaapTempResponse), }, nil } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 526cbac8870..a0387fa7ecc 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -177,7 +177,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { return ret, nil } -func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest, response ResponseRequest) error { +func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { @@ -285,17 +285,15 @@ func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { return nil } -func (w *WaapRuntimeConfig) ProcessInBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { +func (w *WaapRuntimeConfig) ProcessInBandRules(request ParsedRequest) error { for _, rule := range w.InBandRules { - interrupt, err := rule.Eval(request) + _, err := rule.Eval(request) if err != nil { - return nil, fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) - } - if interrupt != nil { - return interrupt, nil + return fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) } + //... } - return nil, nil + return nil } func (w *WaapRuntimeConfig) ProcessOutOfBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { From 6a47b9e97d7c69d957e8900b483808c44befeda9 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 13 Sep 2023 18:03:03 +0200 Subject: [PATCH 079/263] up --- pkg/acquisition/modules/waap/waap.go | 24 +------------------- pkg/acquisition/modules/waap/waap_runner.go | 1 + pkg/waf/env.go | 2 +- pkg/waf/waap.go | 25 +++++++++++++++++---- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 213f9d67cdf..3f2287e2f96 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -267,29 +267,7 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel - - // //@tko this parts needs to be redone - // if message.Err != nil { - // log.Errorf("Error while processing InBAND: %s", err) - // rw.WriteHeader(http.StatusInternalServerError) - // return - // } - - // //here we must rely on WaapRuntimeConfig to know what to do - // if message.Interruption != nil { - // rw.WriteHeader(http.StatusForbidden) - // action := message.Interruption.Action - // if action == "deny" { // bouncers understand "ban" and not "deny" - // action = "ban" - // } - // body, err := json.Marshal(BodyResponse{Action: action}) - // if err != nil { - // log.Errorf("unable to build response: %s", err) - // } else { - // rw.Write(body) - // } - // return - // } + log.Infof("resp %+v", response) rw.WriteHeader(response.HTTPResponseCode) body, err := json.Marshal(BodyResponse{Action: response.Action}) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 7d43c30e7a7..ef44903949f 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -43,6 +43,7 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Errorf("unable to process PreEval rules: %s", err) continue } + log.Infof("now response is -> %s", r.WaapRuntime.Response.Action) //inband WAAP rules err = r.WaapRuntime.ProcessInBandRules(request) elapsed := time.Since(startParsing) diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 7e5f78c3546..0ac474e893b 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -26,7 +26,7 @@ func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { } // simply used to ease the compilation & runtime of the hooks -func GetHookEnv(w WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { +func GetHookEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { return map[string]interface{}{ "inband_rules": w.InBandRules, "outband_rules": w.OutOfBandRules, diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index a0387fa7ecc..12adba1478a 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -32,7 +32,7 @@ func (h *Hook) Build() error { h.FilterExpr = program } for _, apply := range h.Apply { - program, err := expr.Compile(apply, GetExprWAFOptions(GetHookEnv(WaapRuntimeConfig{}, ParsedRequest{}))...) + program, err := expr.Compile(apply, GetExprWAFOptions(GetHookEnv(&WaapRuntimeConfig{}, ParsedRequest{}))...) if err != nil { return fmt.Errorf("unable to compile apply %s : %w", apply, err) } @@ -220,7 +220,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetHookEnv(*w, request)) + output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, request)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -237,7 +237,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetHookEnv(*w, request)) + _, err := expr.Run(applyExpr, GetHookEnv(w, request)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue @@ -275,7 +275,24 @@ func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { } func (w *WaapRuntimeConfig) SetAction(action string) error { - w.Response.Action = action + log.Infof("setting to %s", action) + switch action { + case "allow": + w.Response.Action = action + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + //how should we handle this ? + case "deny", "ban", "block": + w.Response.Action = "ban" + w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode + case "log": + w.Response.Action = action + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + case "captcha": + w.Response.Action = action + w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode + default: + return fmt.Errorf("unknown action %s", action) + } return nil } From a8321b5cc5ea4600556c195c123e338057678512 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 14 Sep 2023 09:43:22 +0200 Subject: [PATCH 080/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 6 ++++-- pkg/waf/waap.go | 10 ++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index ef44903949f..6ed13115e2c 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -46,13 +46,15 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { log.Infof("now response is -> %s", r.WaapRuntime.Response.Action) //inband WAAP rules err = r.WaapRuntime.ProcessInBandRules(request) + if err != nil { + r.logger.Errorf("unable to process InBand rules: %s", err) + continue + } elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) //generate reponse for the remediation component, based on the WAAP config + inband rules evaluation //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation - //response := waf.NewResponseRequest(r.WaapRuntime.InBandTx.Tx, interrupt, request.UUID, err) - err = r.WaapRuntime.ProcessOnMatchRules(request) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 12adba1478a..a093a34ecd0 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -65,6 +65,7 @@ type WaapRuntimeConfig struct { OutOfBandTx ExtendedTransaction //is it a good idea ? InBandTx ExtendedTransaction //is it a good idea ? Response WaapTempResponse + //should we store matched rules here ? } type WaapConfig struct { @@ -181,10 +182,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, map[string]interface{}{ - //"rules": rules, //is it still useful ? - "req": request, - }) + output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, request)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -275,12 +273,12 @@ func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { } func (w *WaapRuntimeConfig) SetAction(action string) error { - log.Infof("setting to %s", action) + //log.Infof("setting to %s", action) switch action { case "allow": w.Response.Action = action w.Response.HTTPResponseCode = w.Config.PassedHTTPCode - //how should we handle this ? + //@tko how should we handle this ? it seems bouncer only understand bans, but it might be misleading ? case "deny", "ban", "block": w.Response.Action = "ban" w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode From 42341222dfa01511d27250bed9757a2677782c7d Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 19 Sep 2023 08:54:31 +0200 Subject: [PATCH 081/263] up --- pkg/waf/env.go | 3 ++- pkg/waf/waap.go | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 0ac474e893b..50a90d5605a 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -34,8 +34,9 @@ func GetHookEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interfac "RemoveInbandRuleByID": w.RemoveInbandRuleByID, "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, "SetAction": w.SetAction, + "SetActionByTag": w.SetActionByTag, "SetHTTPCode": w.SetHTTPCode, - "SetActionByID": w.SetActionnByID, + "SetActionByID": w.SetActionByID, "CancelEvent": w.CancelEvent, } } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index a093a34ecd0..0c4f4e232fc 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -51,8 +51,9 @@ type WaapTempResponse struct { // runtime version of WaapConfig type WaapRuntimeConfig struct { - Name string - OutOfBandRules []WaapCollection + Name string + OutOfBandRules []WaapCollection + //OutOfBandEngine XXX InBandRules []WaapCollection DefaultRemediation string CompiledOnLoad []Hook @@ -263,7 +264,12 @@ func (w *WaapRuntimeConfig) CancelEvent() error { return nil } -func (w *WaapRuntimeConfig) SetActionnByID(id int, action string) error { +func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { + panic("not implemented") + return nil +} + +func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { panic("not implemented") return nil } From 502e21bc5bb3ee246e2a27333c06e5a27d488341 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 31 Aug 2023 14:59:37 +0200 Subject: [PATCH 082/263] wip --- pkg/types/constants.go | 2 +- pkg/waf/waf_rule.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 pkg/waf/waf_rule.go diff --git a/pkg/types/constants.go b/pkg/types/constants.go index c82e9a06c6a..fa50b64f367 100644 --- a/pkg/types/constants.go +++ b/pkg/types/constants.go @@ -9,7 +9,7 @@ const PAPIVersion = "v1" const PAPIPollUrl = "/decisions/stream/poll" const PAPIPermissionsUrl = "/permissions" -const CAPIBaseURL = "https://api.dev.crowdsec.net/" +const CAPIBaseURL = "https://api.crowdsec.net/" const CscliOrigin = "cscli" const CrowdSecOrigin = "crowdsec" diff --git a/pkg/waf/waf_rule.go b/pkg/waf/waf_rule.go new file mode 100644 index 00000000000..146d79bbf8e --- /dev/null +++ b/pkg/waf/waf_rule.go @@ -0,0 +1,25 @@ +package waf + +type VPatchRule struct { + //Those 2 together represent something like ARGS.foo + //If only target is set, it's used for variables that are not a collection (REQUEST_METHOD, etc) + Target string `yaml:"target"` + Variable string `yaml:"var` + + //Operations + Match string `yaml:"match"` //@rx + Equals string `yaml:"equals"` //@eq + Transform string `yaml:"transform"` //t:lowercase, t:uppercase, etc + Detect string `yaml:"detect"` //@detectXSS, @detectSQLi, etc + + RulesOr []VPatchRule `yaml:"rules_or"` + RulesAnd []VPatchRule `yaml:"rules_and"` +} + +func (v *VPatchRule) String() string { + //ret := "SecRule " + + if v.Target != "" { + } + return "" +} From ca930cce0996c023dc45f00f36c6b94182bfc9dd Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 11 Sep 2023 10:35:44 +0200 Subject: [PATCH 083/263] wip --- pkg/waf/waf_rule.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/waf/waf_rule.go b/pkg/waf/waf_rule.go index 146d79bbf8e..7a3ad40ad7d 100644 --- a/pkg/waf/waf_rule.go +++ b/pkg/waf/waf_rule.go @@ -4,7 +4,7 @@ type VPatchRule struct { //Those 2 together represent something like ARGS.foo //If only target is set, it's used for variables that are not a collection (REQUEST_METHOD, etc) Target string `yaml:"target"` - Variable string `yaml:"var` + Variable string `yaml:"var"` //Operations Match string `yaml:"match"` //@rx From 7fdd4d04fe6cdfa15ee6044b2969ebf9ec8f4528 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 13 Sep 2023 10:51:44 +0200 Subject: [PATCH 084/263] up --- pkg/acquisition/modules/waap/waap.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 3f2287e2f96..44573677f11 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -49,6 +49,8 @@ type WaapSource struct { InChan chan waf.ParsedRequest WaapRuntime *waf.WaapRuntimeConfig + WaapConfigs map[string]waf.WaapConfig + WaapRunners []WaapRunner //one for each go-routine } From d5e0c8a36b77552e41f37e70facdfaa4bb713dff Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 14 Sep 2023 09:39:24 +0200 Subject: [PATCH 085/263] up --- pkg/waf/waap.go | 1 - pkg/waf/waap_rules_collection.go | 61 ++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 0c4f4e232fc..f7128e991c6 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -117,7 +117,6 @@ func (wc *WaapConfig) Load(file string) error { wc.DefaultPassAction = "allow" } return nil - } func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index ce7e605eced..c641a239733 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -1,20 +1,75 @@ package waf -import corazatypes "github.com/crowdsecurity/coraza/v3/types" +import ( + "fmt" + "os" + + corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "gopkg.in/yaml.v2" + + log "github.com/sirupsen/logrus" +) // to be filled w/ seb update type WaapCollection struct { + collectionName string } // to be filled w/ seb update type WaapCollectionConfig struct { + Type string `yaml:"type"` + Name string `yaml:"name"` SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` MergedRules []string `yaml:"-"` } func LoadCollection(collection string) (WaapCollection, error) { - return WaapCollection{}, nil + + //FIXME: do it once globally + var waapRules map[string]WaapCollectionConfig + for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { + if !hubWafRuleItem.Installed { + continue + } + + content, err := os.ReadFile(hubWafRuleItem.LocalPath) + + if err != nil { + log.Warnf("unable to read file %s : %s", hubWafRuleItem.LocalPath, err) + continue + } + + var rule WaapCollectionConfig + + err = yaml.Unmarshal(content, &rule) + + if err != nil { + log.Warnf("unable to unmarshal file %s : %s", hubWafRuleItem.LocalPath, err) + continue + } + + if rule.Type != "waap-rule" { + log.Warnf("unexpected type %s instead of waap-rule for file %s", rule.Type, hubWafRuleItem.LocalPath) + continue + } + waapRules[rule.Name] = rule + } + + if len(waapRules) == 0 { + return WaapCollection{}, fmt.Errorf("no waap rules found in hub") + } + + var loadedRule WaapCollectionConfig + + if loadedRule, ok := waapRules[collection]; !ok { + return WaapCollection{}, fmt.Errorf("no waap rules found for collection %s", collection) + } + + return WaapCollection{ + collectionName: loadedRule.Name, + }, nil } func (wcc WaapCollectionConfig) LoadCollection(collection string) (WaapCollection, error) { @@ -30,5 +85,5 @@ func (w WaapCollection) Eval(req ParsedRequest) (*corazatypes.Interruption, erro } func (w WaapCollection) GetDisplayName() string { - return "rule XX" + return w.collectionName } From d3ce4cbf8ef370b7b4275dd02ea76715d70aede2 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 14 Sep 2023 11:18:33 +0200 Subject: [PATCH 086/263] up --- pkg/acquisition/modules/waap/waap.go | 2 +- pkg/waf/waap.go | 28 ++++++++++------- pkg/waf/waap_rules_collection.go | 45 ++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 44573677f11..0086c96ea7b 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -134,7 +134,7 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { //let's load the associated waap_config: if w.config.WaapConfigPath != "" { - waapCfg := waf.WaapConfig{} + waapCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "waap_config")} err := waapCfg.Load(w.config.WaapConfigPath) if err != nil { return fmt.Errorf("unable to load waap_config : %s", err) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index f7128e991c6..bd6f4b6de9d 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -70,17 +70,18 @@ type WaapRuntimeConfig struct { } type WaapConfig struct { - Name string `yaml:"name"` - OutOfBandRules []string `yaml:"outofband_rules"` - InBandRules []string `yaml:"inband_rules"` - DefaultRemediation string `yaml:"default_remediation"` - DefaultPassAction string `yaml:"default_pass_action"` - BlockedHTTPCode int `yaml:"blocked_http_code"` - PassedHTTPCode int `yaml:"passed_http_code"` - OnLoad []Hook `yaml:"on_load"` - PreEval []Hook `yaml:"pre_eval"` - OnMatch []Hook `yaml:"on_match"` - VariablesTracking []string `yaml:"variables_tracking"` + Name string `yaml:"name"` + OutOfBandRules []string `yaml:"outofband_rules"` + InBandRules []string `yaml:"inband_rules"` + DefaultRemediation string `yaml:"default_remediation"` + DefaultPassAction string `yaml:"default_pass_action"` + BlockedHTTPCode int `yaml:"blocked_http_code"` + PassedHTTPCode int `yaml:"passed_http_code"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + OnMatch []Hook `yaml:"on_match"` + VariablesTracking []string `yaml:"variables_tracking"` + Logger *log.Entry `yaml:"-"` } func (w *WaapRuntimeConfig) ClearResponse() { @@ -93,6 +94,9 @@ func (w *WaapRuntimeConfig) ClearResponse() { } func (wc *WaapConfig) Load(file string) error { + + wc.Logger.Debugf("loading config %s", file) + yamlFile, err := os.ReadFile(file) if err != nil { return fmt.Errorf("unable to read file %s : %s", file, err) @@ -127,6 +131,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { //load rules for _, rule := range wc.OutOfBandRules { + wc.Logger.Debugf("loading outofband rule %s", rule) collection, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) @@ -135,6 +140,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { } for _, rule := range wc.InBandRules { + wc.Logger.Debugf("loading inband rule %s", rule) collection, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index c641a239733..fd291e29b63 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -3,8 +3,11 @@ package waf import ( "fmt" "os" + "path/filepath" + "strings" corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "gopkg.in/yaml.v2" @@ -14,6 +17,7 @@ import ( // to be filled w/ seb update type WaapCollection struct { collectionName string + Rules []string } // to be filled w/ seb update @@ -22,14 +26,15 @@ type WaapCollectionConfig struct { Name string `yaml:"name"` SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` - MergedRules []string `yaml:"-"` } func LoadCollection(collection string) (WaapCollection, error) { //FIXME: do it once globally - var waapRules map[string]WaapCollectionConfig + waapRules := make(map[string]WaapCollectionConfig) + for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { + log.Infof("loading %s", hubWafRuleItem.LocalPath) if !hubWafRuleItem.Installed { continue } @@ -50,10 +55,11 @@ func LoadCollection(collection string) (WaapCollection, error) { continue } - if rule.Type != "waap-rule" { + if rule.Type != "waf-rule" { //FIXME: rename to waap-rule when hub is properly updated log.Warnf("unexpected type %s instead of waap-rule for file %s", rule.Type, hubWafRuleItem.LocalPath) continue } + log.Infof("Adding %s to waap rules", rule.Name) waapRules[rule.Name] = rule } @@ -62,14 +68,41 @@ func LoadCollection(collection string) (WaapCollection, error) { } var loadedRule WaapCollectionConfig + var ok bool - if loadedRule, ok := waapRules[collection]; !ok { + if loadedRule, ok = waapRules[collection]; !ok { return WaapCollection{}, fmt.Errorf("no waap rules found for collection %s", collection) } - return WaapCollection{ + waapCol := WaapCollection{ collectionName: loadedRule.Name, - }, nil + } + + if loadedRule.SecLangFilesRules != nil { + for _, rulesFile := range loadedRule.SecLangFilesRules { + fullPath := filepath.Join(csconfig.DataDir, rulesFile) + c, err := os.ReadFile(fullPath) + if err != nil { + log.Errorf("unable to read file %s : %s", rulesFile, err) + continue + } + for _, line := range strings.Split(string(c), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + waapCol.Rules = append(waapCol.Rules, line) + } + } + } + + if loadedRule.SecLangRules != nil { + waapCol.Rules = append(waapCol.Rules, loadedRule.SecLangRules...) + } + + return waapCol, nil } func (wcc WaapCollectionConfig) LoadCollection(collection string) (WaapCollection, error) { From 535738b962ca2e48a54940ebb46a48ea7f71e1d4 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 19 Sep 2023 13:16:33 +0200 Subject: [PATCH 087/263] up --- pkg/acquisition/modules/waap/waap.go | 12 +- pkg/acquisition/modules/waap/waap_runner.go | 120 +++++++++++++++++++- pkg/waf/waap.go | 38 ++----- pkg/waf/waap_rules_collection.go | 8 ++ 4 files changed, 145 insertions(+), 33 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 0086c96ea7b..5e7c3e632b8 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -180,6 +180,10 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { logger: wafLogger, WaapRuntime: &wrt, } + err := runner.Init() + if err != nil { + return fmt.Errorf("unable to initialize runner : %s", err) + } w.WaapRunners[nbRoutine] = runner //most likely missign somethign here to actually start the runner :) } @@ -269,10 +273,14 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel + + waapResponse := w.WaapRuntime.GenerateResponse(response.InBandInterrupt) + log.Infof("resp %+v", response) + log.Infof("waap resp %+v", waapResponse) - rw.WriteHeader(response.HTTPResponseCode) - body, err := json.Marshal(BodyResponse{Action: response.Action}) + rw.WriteHeader(waapResponse.HTTPStatus) + body, err := json.Marshal(BodyResponse{Action: waapResponse.Action}) if err != nil { log.Errorf("unable to marshal response: %s", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 6ed13115e2c..f35692e3afd 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -1,9 +1,14 @@ package wafacquisition import ( + "fmt" + "os" "time" "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/experimental" + corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" @@ -22,6 +27,114 @@ type WaapRunner struct { logger *log.Entry } +func (r *WaapRunner) Init() error { + var err error + fs := os.DirFS(csconfig.DataDir) + + inBandRules := "" + outOfBandRules := "" + + for _, collection := range r.WaapRuntime.InBandRules { + inBandRules += collection.String() + } + + for _, collection := range r.WaapRuntime.OutOfBandRules { + outOfBandRules += collection.String() + } + + r.WaapInbandEngine, err = coraza.NewWAF( + coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs), + ) + + if err != nil { + return fmt.Errorf("unable to initialize inband engine : %w", err) + } + + r.WaapOutbandEngine, err = coraza.NewWAF( + coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs), + ) + + if err != nil { + return fmt.Errorf("unable to initialize outband engine : %w", err) + } + + return nil +} + +func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { + var in *corazatypes.Interruption + var err error + + tx := r.WaapInbandEngine.NewTransactionWithID(request.UUID) + + request.Tx = tx.(experimental.FullTransaction) + + if request.Tx.IsRuleEngineOff() { + r.logger.Debugf("rule engine is off, skipping") + return nil + } + + defer func() { + request.Tx.ProcessLogging() + //We don't close the transaction here, as it will reset coraza internal state and break out of bands rules + }() + + request.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) + request.Tx.ProcessURI(request.URI, request.Method, request.Proto) //TODO: The doc mentions that GET args needs to be added, but we never call AddArguments ? + + for k, vr := range request.Headers { + for _, v := range vr { + request.Tx.AddRequestHeader(k, v) + } + } + + if request.ClientHost != "" { + request.Tx.AddRequestHeader("Host", request.ClientHost) + request.Tx.SetServerName(request.ClientHost) + } + + if request.TransferEncoding != nil { + request.Tx.AddRequestHeader("Transfer-Encoding", request.TransferEncoding[0]) + } + + in = request.Tx.ProcessRequestHeaders() + + if in != nil { + r.logger.Infof("inband rules matched for headers : %d", in.Action) + return nil + } + + if request.Body != nil && len(request.Body) > 0 { + in, _, err = request.Tx.WriteRequestBody(request.Body) + if err != nil { + r.logger.Errorf("unable to write request body : %s", err) + return err + } + if in != nil { + return nil + } + } + + in, err = request.Tx.ProcessRequestBody() + + if err != nil { + r.logger.Errorf("unable to process request body : %s", err) + return err + } + + if in != nil { + r.logger.Infof("inband rules matched for body : %d", in.RuleID) + return nil + } + + return nil +} + +func (r *WaapRunner) ProcessOutOfBandRules(request waf.ParsedRequest) (*corazatypes.Interruption, error) { + + return nil, nil +} + func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Infof("Waap Runner ready to process event") for { @@ -45,11 +158,16 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { } log.Infof("now response is -> %s", r.WaapRuntime.Response.Action) //inband WAAP rules - err = r.WaapRuntime.ProcessInBandRules(request) + err = r.ProcessInBandRules(&request) if err != nil { r.logger.Errorf("unable to process InBand rules: %s", err) continue } + + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("inband rules matched : %d", in.RuleID) + r.WaapRuntime.Response.InBandInterrupt = true + } elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index bd6f4b6de9d..4100c760f42 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -7,7 +7,6 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - corazatypes "github.com/crowdsecurity/coraza/v3/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -131,7 +130,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { //load rules for _, rule := range wc.OutOfBandRules { - wc.Logger.Debugf("loading outofband rule %s", rule) + wc.Logger.Infof("loading outofband rule %s", rule) collection, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) @@ -139,8 +138,9 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret.OutOfBandRules = append(ret.OutOfBandRules, collection) } + wc.Logger.Infof("Loaded %d outofband rules", len(ret.OutOfBandRules)) for _, rule := range wc.InBandRules { - wc.Logger.Debugf("loading inband rule %s", rule) + wc.Logger.Infof("loading inband rule %s", rule) collection, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) @@ -148,6 +148,8 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret.InBandRules = append(ret.InBandRules, collection) } + wc.Logger.Infof("Loaded %d inband rules", len(ret.InBandRules)) + //load hooks for _, hook := range wc.OnLoad { err := hook.Build() @@ -311,45 +313,21 @@ func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { return nil } -func (w *WaapRuntimeConfig) ProcessInBandRules(request ParsedRequest) error { - for _, rule := range w.InBandRules { - _, err := rule.Eval(request) - if err != nil { - return fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) - } - //... - } - return nil -} - -func (w *WaapRuntimeConfig) ProcessOutOfBandRules(request ParsedRequest) (*corazatypes.Interruption, error) { - for _, rule := range w.OutOfBandRules { - interrupt, err := rule.Eval(request) - if err != nil { - return nil, fmt.Errorf("unable to process inband rule %s : %s", rule.GetDisplayName(), err) - } - if interrupt != nil { - return interrupt, nil - } - } - return nil, nil -} - type BodyResponse struct { Action string `json:"action"` HTTPStatus int `json:"http_status"` } -func (w *WaapRuntimeConfig) GenerateResponse(interrupted bool) (BodyResponse, error) { +func (w *WaapRuntimeConfig) GenerateResponse(interrupted bool) BodyResponse { resp := BodyResponse{} //if there is no interrupt, we should allow with default code if !interrupted { resp.Action = w.Config.DefaultPassAction resp.HTTPStatus = w.Config.PassedHTTPCode - return resp, nil + return resp } resp.Action = w.Config.DefaultRemediation resp.HTTPStatus = w.Config.BlockedHTTPCode - return resp, nil + return resp } diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index fd291e29b63..f43564b31ae 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -120,3 +120,11 @@ func (w WaapCollection) Eval(req ParsedRequest) (*corazatypes.Interruption, erro func (w WaapCollection) GetDisplayName() string { return w.collectionName } + +func (w WaapCollection) String() string { + ret := "" + for _, rule := range w.Rules { + ret += rule + "\n" + } + return ret +} From dd7fa825431ec5f1a669e238b5282b6bdba56ceb Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 3 Oct 2023 10:31:21 +0200 Subject: [PATCH 088/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 33 +++++++++++++++------ 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index f35692e3afd..a90517f6fdb 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -61,13 +61,10 @@ func (r *WaapRunner) Init() error { return nil } -func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { +func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *waf.ParsedRequest) error { var in *corazatypes.Interruption var err error - - tx := r.WaapInbandEngine.NewTransactionWithID(request.UUID) - - request.Tx = tx.(experimental.FullTransaction) + request.Tx = tx if request.Tx.IsRuleEngineOff() { r.logger.Debugf("rule engine is off, skipping") @@ -76,7 +73,7 @@ func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { defer func() { request.Tx.ProcessLogging() - //We don't close the transaction here, as it will reset coraza internal state and break out of bands rules + //We don't close the transaction here, as it will reset coraza internal state and break variable tracking }() request.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) @@ -123,16 +120,23 @@ func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { } if in != nil { - r.logger.Infof("inband rules matched for body : %d", in.RuleID) + r.logger.Infof("rules matched for body : %d", in.RuleID) return nil } return nil } -func (r *WaapRunner) ProcessOutOfBandRules(request waf.ParsedRequest) (*corazatypes.Interruption, error) { +func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { + tx := r.WaapInbandEngine.NewTransactionWithID(request.UUID) + err := r.processRequest(tx.(experimental.FullTransaction), request) + return err +} - return nil, nil +func (r *WaapRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { + tx := r.WaapOutbandEngine.NewTransactionWithID(request.UUID) + err := r.processRequest(tx.(experimental.FullTransaction), request) + return err } func (r *WaapRunner) Run(t *tomb.Tomb) error { @@ -182,6 +186,17 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- r.WaapRuntime.Response + err = r.ProcessOutOfBandRules(&request) + if err != nil { + r.logger.Errorf("unable to process OutOfBand rules: %s", err) + continue + } + + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("outband rules matched : %d", in.RuleID) + r.WaapRuntime.Response.OutOfBandInterrupt = true + } + } } } From 92a3c4b2fb7927fdf8ffadd1c2bd7feea14dba2f Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 4 Oct 2023 14:17:21 +0200 Subject: [PATCH 089/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 25 ++++++++++++++++----- pkg/waf/env.go | 2 ++ pkg/waf/request.go | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index a90517f6fdb..e20fd918f1d 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -150,6 +150,9 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Infof("Requests handled by runner %s", request.UUID) r.WaapRuntime.ClearResponse() + request.IsInBand = true + request.IsOutBand = false + WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() //to measure the time spent in the WAF startParsing := time.Now() @@ -171,21 +174,25 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("inband rules matched : %d", in.RuleID) r.WaapRuntime.Response.InBandInterrupt = true + + err = r.WaapRuntime.ProcessOnMatchRules(request) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + continue + } } elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) //generate reponse for the remediation component, based on the WAAP config + inband rules evaluation //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation - err = r.WaapRuntime.ProcessOnMatchRules(request) - if err != nil { - r.logger.Errorf("unable to process OnMatch rules: %s", err) - continue - } // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- r.WaapRuntime.Response + request.IsInBand = false + request.IsOutBand = true + err = r.ProcessOutOfBandRules(&request) if err != nil { r.logger.Errorf("unable to process OutOfBand rules: %s", err) @@ -195,6 +202,14 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("outband rules matched : %d", in.RuleID) r.WaapRuntime.Response.OutOfBandInterrupt = true + } else { + continue + } + + err = r.WaapRuntime.ProcessOnMatchRules(request) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + continue } } diff --git a/pkg/waf/env.go b/pkg/waf/env.go index 50a90d5605a..2322a6e43af 100644 --- a/pkg/waf/env.go +++ b/pkg/waf/env.go @@ -38,5 +38,7 @@ func GetHookEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interfac "SetHTTPCode": w.SetHTTPCode, "SetActionByID": w.SetActionByID, "CancelEvent": w.CancelEvent, + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, } } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 6fd95834ef2..49fcfe45134 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -74,6 +74,8 @@ type ParsedRequest struct { UUID string Tx experimental.FullTransaction ResponseChannel chan WaapTempResponse + IsInBand bool + IsOutBand bool } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine From d3bb9f8ae1fab46ade6ccd921c498604b029c673 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 17 Oct 2023 09:32:40 +0200 Subject: [PATCH 090/263] up --- pkg/waf/waap_rule.go | 104 ++++++++++++++++++++++++++++++ pkg/waf/waap_rule_test.go | 107 +++++++++++++++++++++++++++++++ pkg/waf/waap_rules_collection.go | 20 ++++-- pkg/waf/waf_rule.go | 25 -------- 4 files changed, 226 insertions(+), 30 deletions(-) create mode 100644 pkg/waf/waap_rule.go create mode 100644 pkg/waf/waap_rule_test.go delete mode 100644 pkg/waf/waf_rule.go diff --git a/pkg/waf/waap_rule.go b/pkg/waf/waap_rule.go new file mode 100644 index 00000000000..5cc934a45f1 --- /dev/null +++ b/pkg/waf/waap_rule.go @@ -0,0 +1,104 @@ +package waf + +import ( + "fmt" + "strings" +) + +type VPatchRule struct { + //Those 2 together represent something like ARGS.foo + //If only target is set, it's used for variables that are not a collection (REQUEST_METHOD, etc) + Target string `yaml:"target"` + Variable string `yaml:"var"` + + Match string `yaml:"match"` //@rx + Equals string `yaml:"equals"` //@eq + Transform string `yaml:"transform"` //t:lowercase, t:uppercase, etc + Detect string `yaml:"detect"` //@detectXSS, @detectSQLi, etc + Logic string `yaml:"logic,omitempty"` // "AND", "OR", or empty if not applicable + SubRules []VPatchRule `yaml:"sub_rules,omitempty"` + + id int +} + +func (v *VPatchRule) String() string { + return strings.Trim(v.constructRule(0), "\n") +} + +func countTotalRules(rules []VPatchRule) int { + count := 0 + for _, rule := range rules { + count++ + if rule.Logic == "AND" { + count += countTotalRules(rule.SubRules) + } + } + return count +} + +func (v *VPatchRule) constructRule(depth int) string { + var result string + result = v.singleRuleString() + + if len(v.SubRules) == 0 { + return result + "\n" + } + + switch v.Logic { + case "AND": + // Add "chain" to the current rule + result = strings.TrimSuffix(result, `"`) + `,chain"` + "\n" + for _, subRule := range v.SubRules { + result += subRule.constructRule(depth + 1) + } + case "OR": + skips := countTotalRules(v.SubRules) - 1 + // If the "OR" rule is at the top level and is followed by any rule, we need to count that too + if depth == 0 { + skips++ // For the current rule + } + // Add the skip directive to the current rule too + result = strings.TrimSuffix(result, `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) + for _, subRule := range v.SubRules { + skips-- + if skips > 0 { + // Append skip directive and decrease the skip count + result += strings.TrimSuffix(subRule.singleRuleString(), `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) + } else { + // If no skip is required, append only a newline + result += subRule.singleRuleString() + "\n" + } + } + } + return result +} + +func (v *VPatchRule) singleRuleString() string { + var operator string + var ruleStr string + + if v.Match != "" { + operator = fmt.Sprintf("@rx %s", v.Match) + } else if v.Equals != "" { + operator = fmt.Sprintf("@eq %s", v.Equals) + } else { + return "" + } + + if v.Variable != "" { + ruleStr = fmt.Sprintf(`SecRule %s:%s "%s"`, v.Target, v.Variable, operator) + } else { + ruleStr = fmt.Sprintf(`SecRule %s "%s"`, v.Target, operator) + } + + actions := fmt.Sprintf(` "id:%d,deny,log`, v.id) + + // Handle transformation + if v.Transform != "" { + actions = actions + fmt.Sprintf(",t:%s", v.Transform) + } + actions = actions + `"` + ruleStr = ruleStr + actions + + return ruleStr +} diff --git a/pkg/waf/waap_rule_test.go b/pkg/waf/waap_rule_test.go new file mode 100644 index 00000000000..59cc7f89773 --- /dev/null +++ b/pkg/waf/waap_rule_test.go @@ -0,0 +1,107 @@ +package waf + +import "testing" + +func TestVPatchRuleString(t *testing.T) { + tests := []struct { + name string + rule VPatchRule + expected string + }{ + { + name: "Base Rule", + rule: VPatchRule{ + Target: "ARGS", + Variable: "foo", + Match: "[^a-zA-Z]", + Transform: "lowercase", + }, + expected: `SecRule ARGS:foo "@rx [^a-zA-Z]" "id:0,deny,log,t:lowercase"`, + }, + { + name: "AND Logic Rule", + rule: VPatchRule{ + Target: "ARGS", + Variable: "bar", + Match: "[0-9]", + Logic: "AND", + SubRules: []VPatchRule{ + { + Target: "REQUEST_URI", + Match: "/joomla/index.php/component/users/", + }, + }, + }, + expected: `SecRule ARGS:bar "@rx [0-9]" "id:0,deny,log,chain" +SecRule REQUEST_URI "@rx /joomla/index.php/component/users/" "id:0,deny,log"`, + }, + { + name: "OR Logic Rule", + rule: VPatchRule{ + Target: "REQUEST_HEADERS", + Variable: "User-Agent", + Match: "BadBot", + Logic: "OR", + SubRules: []VPatchRule{ + { + Target: "REQUEST_HEADERS", + Variable: "Referer", + Match: "EvilReferer", + }, + { + Target: "REQUEST_METHOD", + Equals: "POST", + }, + }, + }, + expected: `SecRule REQUEST_HEADERS:User-Agent "@rx BadBot" "id:0,deny,log,skip:2" +SecRule REQUEST_HEADERS:Referer "@rx EvilReferer" "id:0,deny,log,skip:1" +SecRule REQUEST_METHOD "@eq POST" "id:0,deny,log"`, + }, + { + name: "AND-OR Logic Mix", + rule: VPatchRule{ + Target: "REQUEST_URI", + Match: "/api/", + Logic: "AND", + SubRules: []VPatchRule{ + { + Target: "ARGS", + Variable: "username", + Match: "admin", + Logic: "OR", + SubRules: []VPatchRule{ + { + Target: "REQUEST_METHOD", + Equals: "POST", + Logic: "AND", + SubRules: []VPatchRule{ + { + Target: "ARGS", + Variable: "action", + Match: "delete", + }, + }, + }, + }, + }, + }, + }, + expected: `SecRule REQUEST_URI "@rx /api/" "id:0,deny,log,chain" +SecRule ARGS:username "@rx admin" "id:0,deny,log,skip:2" +SecRule REQUEST_METHOD "@eq POST" "id:0,deny,log,chain" +SecRule ARGS:action "@rx delete" "id:0,deny,log"`, + }, + // Additional OR test case would be here, but note that the OR logic representation with `skip` is very simplistic. + // It may not be robust enough for complex OR rules in a real-world ModSecurity setup. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.rule.String() + if actual != tt.expected { + t.Errorf("Expected:\n%s\nGot:\n%s", tt.expected, actual) + } + }) + } +} diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index f43564b31ae..0e3cc9eddf5 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -22,10 +22,11 @@ type WaapCollection struct { // to be filled w/ seb update type WaapCollectionConfig struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - SecLangFilesRules []string `yaml:"seclang_files_rules"` - SecLangRules []string `yaml:"seclang_rules"` + Type string `yaml:"type"` + Name string `yaml:"name"` + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + Rules []VPatchRule `yaml:"rules"` } func LoadCollection(collection string) (WaapCollection, error) { @@ -48,7 +49,7 @@ func LoadCollection(collection string) (WaapCollection, error) { var rule WaapCollectionConfig - err = yaml.Unmarshal(content, &rule) + err = yaml.UnmarshalStrict(content, &rule) if err != nil { log.Warnf("unable to unmarshal file %s : %s", hubWafRuleItem.LocalPath, err) @@ -74,6 +75,8 @@ func LoadCollection(collection string) (WaapCollection, error) { return WaapCollection{}, fmt.Errorf("no waap rules found for collection %s", collection) } + log.Infof("Found rule collection %s with %+v", loadedRule.Name, loadedRule) + waapCol := WaapCollection{ collectionName: loadedRule.Name, } @@ -102,6 +105,13 @@ func LoadCollection(collection string) (WaapCollection, error) { waapCol.Rules = append(waapCol.Rules, loadedRule.SecLangRules...) } + if loadedRule.Rules != nil { + for _, rule := range loadedRule.Rules { + log.Infof("Adding rule %s", rule.String()) + waapCol.Rules = append(waapCol.Rules, rule.String()) + } + } + return waapCol, nil } diff --git a/pkg/waf/waf_rule.go b/pkg/waf/waf_rule.go deleted file mode 100644 index 7a3ad40ad7d..00000000000 --- a/pkg/waf/waf_rule.go +++ /dev/null @@ -1,25 +0,0 @@ -package waf - -type VPatchRule struct { - //Those 2 together represent something like ARGS.foo - //If only target is set, it's used for variables that are not a collection (REQUEST_METHOD, etc) - Target string `yaml:"target"` - Variable string `yaml:"var"` - - //Operations - Match string `yaml:"match"` //@rx - Equals string `yaml:"equals"` //@eq - Transform string `yaml:"transform"` //t:lowercase, t:uppercase, etc - Detect string `yaml:"detect"` //@detectXSS, @detectSQLi, etc - - RulesOr []VPatchRule `yaml:"rules_or"` - RulesAnd []VPatchRule `yaml:"rules_and"` -} - -func (v *VPatchRule) String() string { - //ret := "SecRule " - - if v.Target != "" { - } - return "" -} From 7db5bf8979beaac6fa52b7fa61883ca62a011d15 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:19:29 +0200 Subject: [PATCH 091/263] pkg/csconfig: set prometheus address:port defaults (#2533) We set these default in one place (after loading the configuration) instead of leaving that to both metric server and consumer. --- cmd/crowdsec-cli/metrics.go | 2 +- cmd/crowdsec-cli/utils.go | 6 ------ cmd/crowdsec/metrics.go | 8 -------- pkg/csconfig/config.go | 15 +++++++++++++++ test/bats/08_metrics.bats | 17 +++++++++-------- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 8ab3f01bd7d..0e4926b7214 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -306,7 +306,7 @@ func runMetrics(cmd *cobra.Command, args []string) error { err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) if err != nil { - return fmt.Errorf("could not fetch prometheus metrics: %w", err) + return err } return nil } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 503653f8268..3cf427c8996 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -214,13 +214,7 @@ func InspectItem(name string, objecitemType string) { //This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus //But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case listenAddr := csConfig.Prometheus.ListenAddr - if listenAddr == "" { - listenAddr = "127.0.0.1" - } listenPort := csConfig.Prometheus.ListenPort - if listenPort == 0 { - listenPort = 6060 - } prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort) log.Debugf("No prometheus URL provided using: %s", prometheusURL) } diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 103becceded..6371a6046f2 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { if !config.Enabled { return } - if config.ListenAddr == "" { - log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'") - config.ListenAddr = "127.0.0.1" - } - if config.ListenPort == 0 { - log.Warning("prometheus is enabled, but the listen port is empty, using '6060'") - config.ListenPort = 6060 - } // Registering prometheus // If in aggregated mode, do not register events associated with a source, to keep the cardinality low diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 0fa0e1d2d9b..25f6f716ddd 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -58,6 +58,21 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool // this is actually the "merged" yaml return nil, "", fmt.Errorf("%s: %w", configFile, err) } + + if cfg.Prometheus == nil { + cfg.Prometheus = &PrometheusCfg{} + } + + if cfg.Prometheus.ListenAddr == "" { + cfg.Prometheus.ListenAddr = "127.0.0.1" + log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr) + } + + if cfg.Prometheus.ListenPort == 0 { + cfg.Prometheus.ListenPort = 6060 + log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort) + } + return &cfg, configData, nil } diff --git a/test/bats/08_metrics.bats b/test/bats/08_metrics.bats index 836e220484a..0275d7fd4a0 100644 --- a/test/bats/08_metrics.bats +++ b/test/bats/08_metrics.bats @@ -25,8 +25,7 @@ teardown() { @test "cscli metrics (crowdsec not running)" { rune -1 cscli metrics # crowdsec is down - assert_stderr --partial "failed to fetch prometheus metrics" - assert_stderr --partial "connect: connection refused" + assert_stderr --partial 'failed to fetch prometheus metrics: executing GET request for URL \"http://127.0.0.1:6060/metrics\" failed: Get \"http://127.0.0.1:6060/metrics\": dial tcp 127.0.0.1:6060: connect: connection refused' } @test "cscli metrics (bad configuration)" { @@ -43,18 +42,20 @@ teardown() { @test "cscli metrics (missing listen_addr)" { config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_addr is empty, defaulting to 127.0.0.1" } @test "cscli metrics (missing listen_port)" { - config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + config_set 'del(.prometheus.listen_port)' + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_port is empty or zero, defaulting to 6060" } @test "cscli metrics (missing prometheus section)" { config_set 'del(.prometheus)' rune -1 cscli metrics - assert_stderr --partial "prometheus section missing, can't show metrics" + assert_stderr --partial "prometheus is not enabled, can't show metrics" } From 734ba46e6a3896b7335562100858eb575dfdbc38 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:53:17 +0200 Subject: [PATCH 092/263] Refact cscli hub/item commands (#2536) * log.Fatal -> fmt.Errorf * lint cmd/crowdsec-cli hub items and split collection commands * cscli collections: add examples * cscli parsers: avoid globals * cscli scenarios: avoid globals * cscli collections, postoverflows: avoid globals * cscli hub: avoid globals * remove unused globals --- cmd/crowdsec-cli/collections.go | 311 ++++++++++++++++++++--------- cmd/crowdsec-cli/config_restore.go | 8 +- cmd/crowdsec-cli/hub.go | 140 +++++++------ cmd/crowdsec-cli/main.go | 5 - cmd/crowdsec-cli/parsers.go | 227 ++++++++++++++------- cmd/crowdsec-cli/postoverflows.go | 229 +++++++++++++++------ cmd/crowdsec-cli/scenarios.go | 281 +++++++++++++++++--------- cmd/crowdsec-cli/support.go | 2 +- pkg/cwhub/helpers.go | 19 +- pkg/cwhub/helpers_test.go | 8 +- pkg/cwhub/loader.go | 6 +- 11 files changed, 836 insertions(+), 400 deletions(-) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 6806d39a712..f81cf4690de 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -12,11 +12,15 @@ import ( ) func NewCollectionsCmd() *cobra.Command { - var cmdCollections = &cobra.Command{ + cmdCollections := &cobra.Command{ Use: "collections [action]", - Short: "Manage collections from hub", - Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`, - /*TBD fix help*/ + Short: "Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.", + Example: `cscli collections install crowdsec/xxx crowdsec/xyz +cscli collections inspect crowdsec/xxx crowdsec/xyz +cscli collections upgrade crowdsec/xxx crowdsec/xyz +cscli collections list +cscli collections remove crowdsec/xxx crowdsec/xyz +`, Args: cobra.MinimumNArgs(1), Aliases: []string{"collection"}, DisableAutoGenTag: true, @@ -35,42 +39,130 @@ func NewCollectionsCmd() *cobra.Command { }, } - var ignoreError bool + cmdCollections.AddCommand(NewCollectionsInstallCmd()) + cmdCollections.AddCommand(NewCollectionsRemoveCmd()) + cmdCollections.AddCommand(NewCollectionsUpgradeCmd()) + cmdCollections.AddCommand(NewCollectionsInspectCmd()) + cmdCollections.AddCommand(NewCollectionsListCmd()) - var cmdCollectionsInstall = &cobra.Command{ - Use: "install collection", - Short: "Install given collection(s)", - Long: `Fetch and install given collection(s) from hub`, - Example: `cscli collections install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), + return cmdCollections +} + +func runCollectionsInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.COLLECTIONS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) + Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewCollectionsInstallCmd() *cobra.Command { + cmdCollectionsInstall := &cobra.Command{ + Use: "install collection", + Short: "Install given collection(s)", + Long: `Fetch and install given collection(s) from hub`, + Example: `cscli collections install crowdsec/xxx crowdsec/xyz`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.COLLECTIONS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.COLLECTIONS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) - Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } + RunE: runCollectionsInstall, + } + + flags := cmdCollectionsInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple collections") + + return cmdCollectionsInstall +} + +func runCollectionsRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one collection to remove or '--all'") + } + + for _, name := range args { + if !force { + item := cwhub.GetItem(cwhub.COLLECTIONS, name) + if item == nil { + return fmt.Errorf("unable to retrieve: %s", name) } - return nil - }, + if len(item.BelongsToCollections) > 0 { + log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections) + log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name) + continue + } + } + + err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force) + if err != nil { + return err + } } - cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections") - cmdCollections.AddCommand(cmdCollectionsInstall) - var cmdCollectionsRemove = &cobra.Command{ + return nil +} + +func NewCollectionsRemoveCmd() *cobra.Command { + cmdCollectionsRemove := &cobra.Command{ Use: "remove collection", Short: "Remove given collection(s)", Long: `Remove given collection(s) from hub`, @@ -80,39 +172,48 @@ func NewCollectionsCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction) - return nil - } + RunE: runCollectionsRemove, + } - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to remove or '--all'") - } + flags := cmdCollectionsRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove : Remove tainted and outdated files") + flags.Bool("all", false, "Delete all the collections") - for _, name := range args { - if !forceAction { - item := cwhub.GetItem(cwhub.COLLECTIONS, name) - if item == nil { - return fmt.Errorf("unable to retrieve: %s", name) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections) - log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name) - continue - } - } - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction) - } - return nil - }, + return cmdCollectionsRemove +} + +func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force) + return nil } - cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections") - cmdCollections.AddCommand(cmdCollectionsRemove) - var cmdCollectionsUpgrade = &cobra.Command{ + if len(args) == 0 { + return fmt.Errorf("specify at least one collection to upgrade or '--all'") + } + + for _, name := range args { + cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force) + } + + return nil +} + +func NewCollectionsUpgradeCmd() *cobra.Command { + cmdCollectionsUpgrade := &cobra.Command{ Use: "upgrade collection", Short: "Upgrade given collection(s)", Long: `Fetch and upgrade given collection(s) from hub`, @@ -121,25 +222,35 @@ func NewCollectionsCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction) - } - } - return nil - }, + RunE: runCollectionsUpgrade, + } + + flags := cmdCollectionsUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the collections") + flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + + return cmdCollectionsUpgrade +} + +func runCollectionsInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + var err error + // XXX: set global + prometheusURL, err = flags.GetString("url") + if err != nil { + return err + } + + for _, name := range args { + InspectItem(name, cwhub.COLLECTIONS) } - cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections") - cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - cmdCollections.AddCommand(cmdCollectionsUpgrade) - var cmdCollectionsInspect = &cobra.Command{ + return nil +} + +func NewCollectionsInspectCmd() *cobra.Command { + cmdCollectionsInspect := &cobra.Command{ Use: "inspect collection", Short: "Inspect given collection", Long: `Inspect given collection`, @@ -149,28 +260,42 @@ func NewCollectionsCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - for _, name := range args { - InspectItem(name, cwhub.COLLECTIONS) - } - }, + RunE: runCollectionsInspect, + } + + flags := cmdCollectionsInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + + return cmdCollectionsInspect +} + +func runCollectionsList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err } - cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - cmdCollections.AddCommand(cmdCollectionsInspect) - var cmdCollectionsList = &cobra.Command{ + // XXX: will happily ignore missing collections + ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) + + return nil +} + +func NewCollectionsListCmd() *cobra.Command { + cmdCollectionsList := &cobra.Command{ Use: "list collection [-a]", Short: "List all collections", Long: `List all collections`, Example: `cscli collections list`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) - }, + RunE: runCollectionsList, } - cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - cmdCollections.AddCommand(cmdCollectionsList) - return cmdCollections + flags := cmdCollectionsList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmdCollectionsList } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 395e943bc35..a528372effb 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -27,10 +27,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if item == nil { return "", fmt.Errorf("error retrieving item") } - if downloadOnly && item.Downloaded && item.UpToDate { - return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil - } - err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false) + err := cwhub.DownloadLatest(csConfig.Hub, item, false, false) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } @@ -38,9 +35,6 @@ func silentInstallItem(name string, obtype string) (string, error) { return "", err } - if downloadOnly { - return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil - } err = cwhub.EnableItem(csConfig.Hub, item) if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 7bdfd5162a8..c9ca62d6798 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -45,33 +45,71 @@ cscli hub update # Download list of available configurations from the hub return cmdHub } +func runHubList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err := require.Hub(csConfig); err != nil { + return err + } + + // use LocalSync to get warnings about tainted / outdated items + warn, _ := cwhub.LocalSync(csConfig.Hub) + for _, v := range warn { + log.Info(v) + } + + cwhub.DisplaySummary() + + ListItems(color.Output, []string{ + cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, + }, args, true, false, all) + + return nil +} + func NewHubListCmd() *cobra.Command { var cmdHubList = &cobra.Command{ Use: "list [-a]", Short: "List installed configs", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } + RunE: runHubList, + } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - cwhub.DisplaySummary() - ListItems(color.Output, []string{ - cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, - }, args, true, false, all) + flags := cmdHubList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") - return nil - }, + return cmdHubList +} + +func runHubUpdate(cmd *cobra.Command, args []string) error { + if err := csConfig.LoadHub(); err != nil { + return err } - cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - return cmdHubList + if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + if !errors.Is(err, cwhub.ErrIndexNotFound) { + return fmt.Errorf("failed to get Hub index : %w", err) + } + log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) + cwhub.HubBranch = "master" + if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + return fmt.Errorf("failed to get Hub index after retry: %w", err) + } + } + + // use LocalSync to get warnings about tainted / outdated items + warn, _ := cwhub.LocalSync(csConfig.Hub) + for _, v := range warn { + log.Info(v) + } + + return nil } func NewHubUpdateCmd() *cobra.Command { @@ -92,33 +130,36 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return nil }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadHub(); err != nil { - return err - } - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - if !errors.Is(err, cwhub.ErrIndexNotFound) { - return fmt.Errorf("failed to get Hub index : %w", err) - } - log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) - cwhub.HubBranch = "master" - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("failed to get Hub index after retry: %w", err) - } - } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - - return nil - }, + RunE: runHubUpdate, } return cmdHubUpdate } +func runHubUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + if err := require.Hub(csConfig); err != nil { + return err + } + + log.Infof("Upgrading collections") + cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force) + log.Infof("Upgrading parsers") + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force) + log.Infof("Upgrading scenarios") + cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force) + log.Infof("Upgrading postoverflows") + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force) + + return nil +} + func NewHubUpgradeCmd() *cobra.Command { var cmdHubUpgrade = &cobra.Command{ Use: "upgrade", @@ -137,24 +178,11 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return nil }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - log.Infof("Upgrading collections") - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - log.Infof("Upgrading parsers") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - log.Infof("Upgrading scenarios") - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - log.Infof("Upgrading postoverflows") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - - return nil - }, + RunE: runHubUpgrade, } - cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + flags := cmdHubUpgrade.Flags() + flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") return cmdHubUpgrade } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index e99fd53ab22..d0320ac36f2 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -29,11 +29,6 @@ var dbClient *database.Client var OutputFormat string var OutputColor string -var downloadOnly bool -var forceAction bool -var purge bool -var all bool - var prometheusURL string var mergedConfig string diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index d97b070db74..c1f33e804eb 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -12,7 +12,7 @@ import ( ) func NewParsersCmd() *cobra.Command { - var cmdParsers = &cobra.Command{ + cmdParsers := &cobra.Command{ Use: "parsers [action] [config]", Short: "Install/Remove/Upgrade/Inspect parser(s) from hub", Example: `cscli parsers install crowdsecurity/sshd-logs @@ -48,10 +48,46 @@ cscli parsers remove crowdsecurity/sshd-logs return cmdParsers } -func NewParsersInstallCmd() *cobra.Command { - var ignoreError bool +func runParsersInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.PARSERS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.PARSERS, name) + Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) + + continue + } - var cmdParsersInstall = &cobra.Command{ + if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewParsersInstallCmd() *cobra.Command { + cmdParsersInstall := &cobra.Command{ Use: "install [config]", Short: "Install given parser(s)", Long: `Fetch and install given parser(s) from hub`, @@ -61,32 +97,58 @@ func NewParsersInstallCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS, name) - Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runParsersInstall, } - cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers") + flags := cmdParsersInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: Overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple parsers") return cmdParsersInstall } +func runParsersRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one parser to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewParsersRemoveCmd() *cobra.Command { cmdParsersRemove := &cobra.Command{ Use: "remove [config]", @@ -98,29 +160,44 @@ func NewParsersRemoveCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction) - return nil - } + RunE: runParsersRemove, + } - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to remove or '--all'") - } + flags := cmdParsersRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: Remove tainted and outdated files") + flags.Bool("all", false, "Delete all the parsers") - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction) - } + return cmdParsersRemove +} - return nil - }, +func runParsersUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err } - cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers") + if all { + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force) + return nil + } - return cmdParsersRemove + if len(args) == 0 { + return fmt.Errorf("specify at least one parser to upgrade or '--all'") + } + + for _, name := range args { + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force) + } + + return nil } func NewParsersUpgradeCmd() *cobra.Command { @@ -133,62 +210,78 @@ func NewParsersUpgradeCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction) - } - } - return nil - }, + RunE: runParsersUpgrade, } - cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers") - cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + flags := cmdParsersUpgrade.Flags() + flags.Bool("all", false, "Upgrade all the parsers") + flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") return cmdParsersUpgrade } +func runParsersInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + var err error + // XXX: set global + prometheusURL, err = flags.GetString("url") + if err != nil { + return err + } + + InspectItem(args[0], cwhub.PARSERS) + + return nil +} + func NewParsersInspectCmd() *cobra.Command { - var cmdParsersInspect = &cobra.Command{ + cmdParsersInspect := &cobra.Command{ Use: "inspect [name]", Short: "Inspect given parser", Long: `Inspect given parser`, Example: `cscli parsers inspect crowdsec/xxx`, - DisableAutoGenTag: true, Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS) - }, + RunE: runParsersInspect, } - cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") + flags := cmdParsersInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") return cmdParsersInspect } +func runParsersList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + // XXX: will happily ignore missing parsers + ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) + + return nil +} + func NewParsersListCmd() *cobra.Command { - var cmdParsersList = &cobra.Command{ + cmdParsersList := &cobra.Command{ Use: "list [name]", Short: "List all parsers or given one", Long: `List all parsers or given one`, Example: `cscli parsers list cscli parser list crowdsecurity/xxx`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) - }, + RunE: runParsersList, } - cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + flags := cmdParsersList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdParsersList } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index f4db0a79e10..6c3aa6c8097 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -16,10 +16,11 @@ func NewPostOverflowsCmd() *cobra.Command { Use: "postoverflows [action] [config]", Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub", Example: `cscli postoverflows install crowdsecurity/cdn-whitelist - cscli postoverflows inspect crowdsecurity/cdn-whitelist - cscli postoverflows upgrade crowdsecurity/cdn-whitelist - cscli postoverflows list - cscli postoverflows remove crowdsecurity/cdn-whitelist`, +cscli postoverflows inspect crowdsecurity/cdn-whitelist +cscli postoverflows upgrade crowdsecurity/cdn-whitelist +cscli postoverflows list +cscli postoverflows remove crowdsecurity/cdn-whitelist +`, Args: cobra.MinimumNArgs(1), Aliases: []string{"postoverflow"}, DisableAutoGenTag: true, @@ -47,9 +48,45 @@ func NewPostOverflowsCmd() *cobra.Command { return cmdPostOverflows } -func NewPostOverflowsInstallCmd() *cobra.Command { - var ignoreError bool +func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name) + Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError) + + continue + } + if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewPostOverflowsInstallCmd() *cobra.Command { cmdPostOverflowsInstall := &cobra.Command{ Use: "install [config]", Short: "Install given postoverflow(s)", @@ -60,32 +97,58 @@ func NewPostOverflowsInstallCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name) - Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runPostOverflowsInstall, } - cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows") + flags := cmdPostOverflowsInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows") return cmdPostOverflowsInstall } +func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one postoverflow to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewPostOverflowsRemoveCmd() *cobra.Command { cmdPostOverflowsRemove := &cobra.Command{ Use: "remove [config]", @@ -97,29 +160,44 @@ func NewPostOverflowsRemoveCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction) - return nil - } + RunE: runPostOverflowsRemove, + } - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to remove or '--all'") - } + flags := cmdPostOverflowsRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove : Remove tainted and outdated files") + flags.Bool("all", false, "Delete all the postoverflows") - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction) - } + return cmdPostOverflowsRemove +} - return nil - }, +func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err } - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows") + all, err := flags.GetBool("all") + if err != nil { + return err + } - return cmdPostOverflowsRemove + if all { + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force) + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") + } + + for _, name := range args { + cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, force) + } + + return nil } func NewPostOverflowsUpgradeCmd() *cobra.Command { @@ -132,46 +210,66 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction) - } - } - return nil - }, + RunE: runPostOverflowUpgrade, } - cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows") - cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + flags := cmdPostOverflowsUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the postoverflows") + flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") return cmdPostOverflowsUpgrade } +func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + var err error + // XXX: set global + prometheusURL, err = flags.GetString("url") + if err != nil { + return err + } + + InspectItem(args[0], cwhub.PARSERS_OVFLW) + + return nil +} + func NewPostOverflowsInspectCmd() *cobra.Command { cmdPostOverflowsInspect := &cobra.Command{ Use: "inspect [config]", Short: "Inspect given postoverflow", Long: `Inspect given postoverflow`, Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS_OVFLW) - }, + RunE: runPostOverflowsInspect, } + flags := cmdPostOverflowsInspect.Flags() + // XXX: is this needed for postoverflows? + flags.StringP("url", "u", "", "Prometheus url") + return cmdPostOverflowsInspect } +func runPostOverflowsList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + // XXX: will happily ignore missing postoverflows + ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) + + return nil +} + func NewPostOverflowsListCmd() *cobra.Command { cmdPostOverflowsList := &cobra.Command{ Use: "list [config]", @@ -180,12 +278,11 @@ func NewPostOverflowsListCmd() *cobra.Command { Example: `cscli postoverflows list cscli postoverflows list crowdsecurity/xxx`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) - }, + RunE: runPostOverflowsList, } - cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + flags := cmdPostOverflowsList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdPostOverflowsList } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 01e0b02dc86..553ffa472cb 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -12,7 +12,7 @@ import ( ) func NewScenariosCmd() *cobra.Command { - var cmdScenarios = &cobra.Command{ + cmdScenarios := &cobra.Command{ Use: "scenarios [action] [config]", Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub", Example: `cscli scenarios list [-a] @@ -48,141 +48,240 @@ cscli scenarios remove crowdsecurity/ssh-bf return cmdScenarios } +func runScenariosInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.SCENARIOS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.SCENARIOS, name) + Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + func NewCmdScenariosInstall() *cobra.Command { - var ignoreError bool - - var cmdScenariosInstall = &cobra.Command{ - Use: "install [config]", - Short: "Install given scenario(s)", - Long: `Fetch and install given scenario(s) from hub`, - Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), + cmdScenariosInstall := &cobra.Command{ + Use: "install [config]", + Short: "Install given scenario(s)", + Long: `Fetch and install given scenario(s) from hub`, + Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.SCENARIOS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.SCENARIOS, name) - Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runScenariosInstall, } - cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios") + + flags := cmdScenariosInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios") return cmdScenariosInstall } +func runScenariosRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one scenario to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewCmdScenariosRemove() *cobra.Command { - var cmdScenariosRemove = &cobra.Command{ - Use: "remove [config]", - Short: "Remove given scenario(s)", - Long: `remove given scenario(s)`, - Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, + cmdScenariosRemove := &cobra.Command{ + Use: "remove [config]", + Short: "Remove given scenario(s)", + Long: `remove given scenario(s)`, + Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction) - return nil - } + RunE: runScenariosRemove, + } - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to remove or '--all'") - } + flags := cmdScenariosRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: Remove tainted and outdated files") + flags.Bool("all", false, "Delete all the scenarios") - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction) - } - return nil - }, + return cmdScenariosRemove +} + +func runScenariosUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err } - cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios") - return cmdScenariosRemove + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force) + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one scenario to upgrade or '--all'") + } + + for _, name := range args { + cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force) + } + + return nil } func NewCmdScenariosUpgrade() *cobra.Command { - var cmdScenariosUpgrade = &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given scenario(s)", - Long: `Fetch and Upgrade given scenario(s) from hub`, - Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`, + cmdScenariosUpgrade := &cobra.Command{ + Use: "upgrade [config]", + Short: "Upgrade given scenario(s)", + Long: `Fetch and Upgrade given scenario(s) from hub`, + Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`, + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction) - } - } - return nil - }, + RunE: runScenariosUpgrade, } - cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios") - cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + flags := cmdScenariosUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the scenarios") + flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") return cmdScenariosUpgrade } +func runScenariosInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + var err error + // XXX: set global + prometheusURL, err = flags.GetString("url") + if err != nil { + return err + } + + InspectItem(args[0], cwhub.SCENARIOS) + + return nil +} + func NewCmdScenariosInspect() *cobra.Command { - var cmdScenariosInspect = &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given scenario", - Long: `Inspect given scenario`, - Example: `cscli scenarios inspect crowdsec/xxx`, - Args: cobra.MinimumNArgs(1), + cmdScenariosInspect := &cobra.Command{ + Use: "inspect [config]", + Short: "Inspect given scenario", + Long: `Inspect given scenario`, + Example: `cscli scenarios inspect crowdsec/xxx`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.SCENARIOS) - }, + RunE: runScenariosInspect, } - cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") + + flags := cmdScenariosInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") return cmdScenariosInspect } +func runScenariosList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + // XXX: will happily ignore missing scenarios + ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) + + return nil +} + func NewCmdScenariosList() *cobra.Command { - var cmdScenariosList = &cobra.Command{ + cmdScenariosList := &cobra.Command{ Use: "list [config]", Short: "List all scenario(s) or given one", Long: `List all scenario(s) or given one`, Example: `cscli scenarios list cscli scenarios list crowdsecurity/xxx`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) - }, + RunE: runScenariosList, } - cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + + flags := cmdScenariosList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdScenariosList } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index e5a4c36abc8..7d80d647153 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -135,7 +135,7 @@ func collectOSInfo() ([]byte, error) { func collectHubItems(itemType string) []byte { out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - ListItems(out, []string{itemType}, []string{}, false, true, all) + ListItems(out, []string{itemType}, []string{}, false, true, false) return out.Bytes() } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index c17e6758d2f..8ffeaa541a1 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -103,29 +103,28 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return nil } -// XXX this must return errors instead of log.Fatal -func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) { +func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error { if name != "" { item := GetItem(itemType, name) if item == nil { - log.Fatalf("unable to retrieve: %s", name) + return fmt.Errorf("unable to retrieve: %s", name) } err := DisableItem(csConfig.Hub, item, purge, forceAction) if err != nil { - log.Fatalf("unable to disable %s : %v", item.Name, err) + return fmt.Errorf("unable to disable %s: %w", item.Name, err) } if err = AddItem(itemType, *item); err != nil { - log.Fatalf("unable to add %s: %v", item.Name, err) + return fmt.Errorf("unable to add %s: %w", item.Name, err) } - return + return nil } if !all { - log.Fatal("removing item: no item specified") + return fmt.Errorf("removing item: no item specified") } disabled := 0 @@ -138,16 +137,18 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo err := DisableItem(csConfig.Hub, &v, purge, forceAction) if err != nil { - log.Fatalf("unable to disable %s : %v", v.Name, err) + return fmt.Errorf("unable to disable %s: %w", v.Name, err) } if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) + return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ } log.Infof("Disabled %d items", disabled) + + return nil } func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index c8bb28c363c..522dead05b0 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -74,7 +74,9 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + require.NoError(t, err) + getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) @@ -122,7 +124,9 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + require.NoError(t, err) + getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 3c45664937c..8b470148c1e 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -98,7 +98,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { //.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) + return itemFileInfo{}, false, fmt.Errorf("path is too short : %s (%d)", path, len(subs)) } ret.fname = subs[len(subs)-1] @@ -108,7 +108,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { } else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) + return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ///.../config/parser/stage/file.yaml ///.../config/postoverflow/stage/file.yaml @@ -352,7 +352,7 @@ func CollecDepsCheck(v *Item) error { for _, subName := range itemSlice { subItem, ok := hubIdx[sliceType][subName] if !ok { - log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name) + return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) } log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) From a00bae60394fcfb58a20ba76151e7b131ed333d8 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:10:11 +0200 Subject: [PATCH 093/263] cmd/crowdsec-cli: remove global prometheusURL (#2542) * cmd/crowdsec-cli: remove global prometheusURL * PrometheusUrl now includes the path (/metrics) --- cmd/crowdsec-cli/collections.go | 8 ++++--- cmd/crowdsec-cli/main.go | 2 -- cmd/crowdsec-cli/metrics.go | 35 +++++++++++++++++++------------ cmd/crowdsec-cli/parsers.go | 8 ++++--- cmd/crowdsec-cli/postoverflows.go | 8 ++++--- cmd/crowdsec-cli/scenarios.go | 8 ++++--- cmd/crowdsec-cli/support.go | 4 ++-- cmd/crowdsec-cli/utils.go | 29 +++++++++---------------- pkg/csconfig/prometheus.go | 2 +- pkg/csconfig/prometheus_test.go | 2 +- 10 files changed, 56 insertions(+), 50 deletions(-) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index f81cf4690de..c3bcf9b7a3f 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -235,13 +235,15 @@ func NewCollectionsUpgradeCmd() *cobra.Command { func runCollectionsInspect(cmd *cobra.Command, args []string) error { flags := cmd.Flags() - var err error - // XXX: set global - prometheusURL, err = flags.GetString("url") + url, err := flags.GetString("url") if err != nil { return err } + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + for _, name := range args { InspectItem(name, cwhub.COLLECTIONS) } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index d0320ac36f2..e80e8cb4142 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -29,8 +29,6 @@ var dbClient *database.Client var OutputFormat string var OutputColor string -var prometheusURL string - var mergedConfig string func initConfig() { diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 0e4926b7214..d8f7d0f8495 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -284,7 +284,23 @@ var noUnit bool func runMetrics(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadPrometheus(); err != nil { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noUnit, err = flags.GetBool("no-unit") + if err != nil { + return err + } + + if err = csConfig.LoadPrometheus(); err != nil { return fmt.Errorf("failed to load prometheus config: %w", err) } @@ -296,16 +312,7 @@ func runMetrics(cmd *cobra.Command, args []string) error { return fmt.Errorf("prometheus is not enabled, can't show metrics") } - if prometheusURL == "" { - prometheusURL = csConfig.Cscli.PrometheusUrl - } - - if prometheusURL == "" { - return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath) - } - - err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) - if err != nil { + if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil { return err } return nil @@ -321,8 +328,10 @@ func NewMetricsCmd() *cobra.Command { DisableAutoGenTag: true, RunE: runMetrics, } - cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") - cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") + + flags := cmdMetrics.PersistentFlags() + flags.StringP("url", "u", "", "Prometheus url (http://:/metrics)") + flags.Bool("no-unit", false, "Show the real number instead of formatted with units") return cmdMetrics } diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index c1f33e804eb..0f7e0d4f8cc 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -223,13 +223,15 @@ func NewParsersUpgradeCmd() *cobra.Command { func runParsersInspect(cmd *cobra.Command, args []string) error { flags := cmd.Flags() - var err error - // XXX: set global - prometheusURL, err = flags.GetString("url") + url, err := flags.GetString("url") if err != nil { return err } + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + InspectItem(args[0], cwhub.PARSERS) return nil diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index 6c3aa6c8097..f94169f56ae 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -223,13 +223,15 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command { func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { flags := cmd.Flags() - var err error - // XXX: set global - prometheusURL, err = flags.GetString("url") + url, err := flags.GetString("url") if err != nil { return err } + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + InspectItem(args[0], cwhub.PARSERS_OVFLW) return nil diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 553ffa472cb..9a9aa825f26 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -223,13 +223,15 @@ func NewCmdScenariosUpgrade() *cobra.Command { func runScenariosInspect(cmd *cobra.Command, args []string) error { flags := cmd.Flags() - var err error - // XXX: set global - prometheusURL, err = flags.GetString("url") + url, err := flags.GetString("url") if err != nil { return err } + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + InspectItem(args[0], cwhub.SCENARIOS) return nil diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 7d80d647153..15c53d0048d 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -69,13 +69,13 @@ func collectMetrics() ([]byte, []byte, error) { } humanMetrics := bytes.NewBuffer(nil) - err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human") + err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human") if err != nil { return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) } - req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil) + req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil) if err != nil { return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 3cf427c8996..82b98cbeff8 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -210,15 +210,6 @@ func InspectItem(name string, objecitemType string) { return } - if prometheusURL == "" { - //This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus - //But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case - listenAddr := csConfig.Prometheus.ListenAddr - listenPort := csConfig.Prometheus.ListenPort - prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort) - log.Debugf("No prometheus URL provided using: %s", prometheusURL) - } - fmt.Printf("\nCurrent metrics : \n") ShowMetrics(hubItem) } @@ -256,18 +247,18 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * func ShowMetrics(hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: - metrics := GetParserMetric(prometheusURL, hubItem.Name) + metrics := GetParserMetric(hubItem.Name) parserMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.SCENARIOS: - metrics := GetScenarioMetric(prometheusURL, hubItem.Name) + metrics := GetScenarioMetric(hubItem.Name) scenarioMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.COLLECTIONS: for _, item := range hubItem.Parsers { - metrics := GetParserMetric(prometheusURL, item) + metrics := GetParserMetric(item) parserMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(prometheusURL, item) + metrics := GetScenarioMetric(item) scenarioMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Collections { @@ -283,10 +274,10 @@ func ShowMetrics(hubItem *cwhub.Item) { } // GetParserMetric is a complete rip from prom2json -func GetParserMetric(url string, itemName string) map[string]map[string]int { +func GetParserMetric(itemName string) map[string]map[string]int { stats := make(map[string]map[string]int) - result := GetPrometheusMetric(url) + result := GetPrometheusMetric() for idx, fam := range result { if !strings.HasPrefix(fam.Name, "cs_") { continue @@ -364,7 +355,7 @@ func GetParserMetric(url string, itemName string) map[string]map[string]int { return stats } -func GetScenarioMetric(url string, itemName string) map[string]int { +func GetScenarioMetric(itemName string) map[string]int { stats := make(map[string]int) stats["instantiation"] = 0 @@ -373,7 +364,7 @@ func GetScenarioMetric(url string, itemName string) map[string]int { stats["pour"] = 0 stats["underflow"] = 0 - result := GetPrometheusMetric(url) + result := GetPrometheusMetric() for idx, fam := range result { if !strings.HasPrefix(fam.Name, "cs_") { continue @@ -419,7 +410,7 @@ func GetScenarioMetric(url string, itemName string) map[string]int { return stats } -func GetPrometheusMetric(url string) []*prom2json.Family { +func GetPrometheusMetric() []*prom2json.Family { mfChan := make(chan *dto.MetricFamily, 1024) // Start with the DefaultTransport for sane defaults. @@ -432,7 +423,7 @@ func GetPrometheusMetric(url string) []*prom2json.Family { go func() { defer trace.CatchPanic("crowdsec/GetPrometheusMetric") - err := prom2json.FetchMetricFamilies(url, mfChan, transport) + err := prom2json.FetchMetricFamilies(csConfig.Cscli.PrometheusUrl, mfChan, transport) if err != nil { log.Fatalf("failed to fetch prometheus metrics : %v", err) } diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index eea768ab798..f92771bb157 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -12,7 +12,7 @@ type PrometheusCfg struct { func (c *Config) LoadPrometheus() error { if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil { if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { - c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) + c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) } } return nil diff --git a/pkg/csconfig/prometheus_test.go b/pkg/csconfig/prometheus_test.go index 79c9ec58f5f..c16002221c6 100644 --- a/pkg/csconfig/prometheus_test.go +++ b/pkg/csconfig/prometheus_test.go @@ -26,7 +26,7 @@ func TestLoadPrometheus(t *testing.T) { }, Cscli: &CscliCfg{}, }, - expectedURL: "http://127.0.0.1:6060", + expectedURL: "http://127.0.0.1:6060/metrics", }, } From f496bd169288df381b490c8ed727edf74b400f3b Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:12:53 +0200 Subject: [PATCH 094/263] bats: more cscli hub tests (#2541) - updated logs and user messages - added func tests for all the items: install, remove, upgrade, list - rewritten taint tests for collections - removed redundant csconfig.LoadPrometheus() --- cmd/crowdsec-cli/collections.go | 84 +++--- cmd/crowdsec-cli/hub.go | 42 +-- cmd/crowdsec-cli/metrics.go | 4 - cmd/crowdsec-cli/parsers.go | 79 +++-- cmd/crowdsec-cli/postoverflows.go | 75 +++-- cmd/crowdsec-cli/scenarios.go | 77 +++-- cmd/crowdsec-cli/support.go | 6 +- cmd/crowdsec-cli/utils.go | 33 +- pkg/csconfig/cscli.go | 8 + pkg/csconfig/cscli_test.go | 15 +- pkg/csconfig/prometheus.go | 11 - pkg/csconfig/prometheus_test.go | 42 --- pkg/cwhub/helpers.go | 18 +- pkg/cwhub/helpers_test.go | 14 +- test/bats/20_collections.bats | 145 --------- test/bats/20_hub_collections.bats | 319 ++++++++++++++++++++ test/bats/20_hub_collections_dep.bats | 86 ++++++ test/bats/20_hub_parsers.bats | 416 ++++++++++++++++++++++++++ test/bats/20_hub_postoverflows.bats | 321 ++++++++++++++++++++ test/bats/20_hub_scenarios.bats | 320 ++++++++++++++++++++ test/lib/setup_file.sh | 6 + 21 files changed, 1739 insertions(+), 382 deletions(-) delete mode 100644 pkg/csconfig/prometheus_test.go delete mode 100644 test/bats/20_collections.bats create mode 100644 test/bats/20_hub_collections.bats create mode 100644 test/bats/20_hub_collections_dep.bats create mode 100644 test/bats/20_hub_parsers.bats create mode 100644 test/bats/20_hub_postoverflows.bats create mode 100644 test/bats/20_hub_scenarios.bats diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index c3bcf9b7a3f..c7a3e2c56fd 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -13,13 +13,13 @@ import ( func NewCollectionsCmd() *cobra.Command { cmdCollections := &cobra.Command{ - Use: "collections [action]", - Short: "Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.", - Example: `cscli collections install crowdsec/xxx crowdsec/xyz -cscli collections inspect crowdsec/xxx crowdsec/xyz -cscli collections upgrade crowdsec/xxx crowdsec/xyz -cscli collections list -cscli collections remove crowdsec/xxx crowdsec/xyz + Use: "collections [collection]...", + Short: "Manage hub collections", + Example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables `, Args: cobra.MinimumNArgs(1), Aliases: []string{"collection"}, @@ -88,10 +88,10 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error { func NewCollectionsInstallCmd() *cobra.Command { cmdCollectionsInstall := &cobra.Command{ - Use: "install collection", + Use: "install ...", Short: "Install given collection(s)", - Long: `Fetch and install given collection(s) from hub`, - Example: `cscli collections install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more collections from hub`, + Example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -102,7 +102,7 @@ func NewCollectionsInstallCmd() *cobra.Command { flags := cmdCollectionsInstall.Flags() flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") flags.Bool("ignore", false, "Ignore errors when installing multiple collections") return cmdCollectionsInstall @@ -143,11 +143,12 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { if !force { item := cwhub.GetItem(cwhub.COLLECTIONS, name) if item == nil { - return fmt.Errorf("unable to retrieve: %s", name) + // XXX: this should be in GetItem? + return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) } if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections) - log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name) + log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections) + log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name) continue } } @@ -163,10 +164,10 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { func NewCollectionsRemoveCmd() *cobra.Command { cmdCollectionsRemove := &cobra.Command{ - Use: "remove collection", + Use: "remove ...", Short: "Remove given collection(s)", - Long: `Remove given collection(s) from hub`, - Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`, + Long: `Remove one or more collections`, + Example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -177,8 +178,8 @@ func NewCollectionsRemoveCmd() *cobra.Command { flags := cmdCollectionsRemove.Flags() flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove : Remove tainted and outdated files") - flags.Bool("all", false, "Delete all the collections") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the collections") return cmdCollectionsRemove } @@ -197,7 +198,9 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { } if all { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + return err + } return nil } @@ -206,7 +209,9 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil { + return err + } } return nil @@ -214,10 +219,10 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { func NewCollectionsUpgradeCmd() *cobra.Command { cmdCollectionsUpgrade := &cobra.Command{ - Use: "upgrade collection", + Use: "upgrade ...", Short: "Upgrade given collection(s)", - Long: `Fetch and upgrade given collection(s) from hub`, - Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more collections from the hub`, + Example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) @@ -227,7 +232,7 @@ func NewCollectionsUpgradeCmd() *cobra.Command { flags := cmdCollectionsUpgrade.Flags() flags.BoolP("all", "a", false, "Upgrade all the collections") - flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdCollectionsUpgrade } @@ -244,8 +249,15 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error { csConfig.Cscli.PrometheusUrl = url } + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + for _, name := range args { - InspectItem(name, cwhub.COLLECTIONS) + if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { + return err + } } return nil @@ -253,10 +265,10 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error { func NewCollectionsInspectCmd() *cobra.Command { cmdCollectionsInspect := &cobra.Command{ - Use: "inspect collection", - Short: "Inspect given collection", - Long: `Inspect given collection`, - Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`, + Use: "inspect ...", + Short: "Inspect given collection(s)", + Long: `Inspect one or more collections`, + Example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -267,6 +279,7 @@ func NewCollectionsInspectCmd() *cobra.Command { flags := cmdCollectionsInspect.Flags() flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdCollectionsInspect } @@ -287,11 +300,12 @@ func runCollectionsList(cmd *cobra.Command, args []string) error { func NewCollectionsListCmd() *cobra.Command { cmdCollectionsList := &cobra.Command{ - Use: "list collection [-a]", - Short: "List all collections", - Long: `List all collections`, - Example: `cscli collections list`, - Args: cobra.ExactArgs(0), + Use: "list [collection... | -a]", + Short: "List collections", + Long: `List of installed/available/specified collections`, + Example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, DisableAutoGenTag: true, RunE: runCollectionsList, } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c9ca62d6798..7b37ec4c7ad 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -15,17 +15,14 @@ import ( func NewHubCmd() *cobra.Command { var cmdHub = &cobra.Command{ Use: "hub [action]", - Short: "Manage Hub", - Long: ` -Hub management + Short: "Manage hub index", + Long: `Hub management List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). -The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. - `, - Example: ` -cscli hub list # List all installed configurations -cscli hub update # Download list of available configurations from the hub - `, +The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`, + Example: `cscli hub list +cscli hub update +cscli hub upgrade`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { @@ -75,7 +72,7 @@ func runHubList(cmd *cobra.Command, args []string) error { func NewHubListCmd() *cobra.Command { var cmdHubList = &cobra.Command{ Use: "list [-a]", - Short: "List installed configs", + Short: "List all installed configurations", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, RunE: runHubList, @@ -115,7 +112,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error { func NewHubUpdateCmd() *cobra.Command { var cmdHubUpdate = &cobra.Command{ Use: "update", - Short: "Fetch available configs from hub", + Short: "Download the latest index (catalog of available configurations)", Long: ` Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. `, @@ -149,13 +146,24 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { } log.Infof("Upgrading collections") - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + return err + } + log.Infof("Upgrading parsers") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + return err + } + log.Infof("Upgrading scenarios") - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + return err + } + log.Infof("Upgrading postoverflows") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil { + return err + } return nil } @@ -163,7 +171,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { func NewHubUpgradeCmd() *cobra.Command { var cmdHubUpgrade = &cobra.Command{ Use: "upgrade", - Short: "Upgrade all configs installed from hub", + Short: "Upgrade all configurations to their latest version", Long: ` Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. `, @@ -182,7 +190,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if } flags := cmdHubUpgrade.Flags() - flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdHubUpgrade } diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index d8f7d0f8495..a03614aae2a 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -300,10 +300,6 @@ func runMetrics(cmd *cobra.Command, args []string) error { return err } - if err = csConfig.LoadPrometheus(); err != nil { - return fmt.Errorf("failed to load prometheus config: %w", err) - } - if csConfig.Prometheus == nil { return fmt.Errorf("prometheus section missing, can't show metrics") } diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index 0f7e0d4f8cc..aee11abd0b9 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -13,13 +13,13 @@ import ( func NewParsersCmd() *cobra.Command { cmdParsers := &cobra.Command{ - Use: "parsers [action] [config]", - Short: "Install/Remove/Upgrade/Inspect parser(s) from hub", - Example: `cscli parsers install crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/sshd-logs -cscli parsers list -cscli parsers remove crowdsecurity/sshd-logs + Use: "parsers [parser]...", + Short: "Manage hub parsers", + Example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs `, Args: cobra.MinimumNArgs(1), Aliases: []string{"parser"}, @@ -88,10 +88,10 @@ func runParsersInstall(cmd *cobra.Command, args []string) error { func NewParsersInstallCmd() *cobra.Command { cmdParsersInstall := &cobra.Command{ - Use: "install [config]", + Use: "install ...", Short: "Install given parser(s)", - Long: `Fetch and install given parser(s) from hub`, - Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more parsers from the hub`, + Example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -102,7 +102,7 @@ func NewParsersInstallCmd() *cobra.Command { flags := cmdParsersInstall.Flags() flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: Overwrite tainted and outdated files") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") flags.Bool("ignore", false, "Ignore errors when installing multiple parsers") return cmdParsersInstall @@ -151,10 +151,10 @@ func runParsersRemove(cmd *cobra.Command, args []string) error { func NewParsersRemoveCmd() *cobra.Command { cmdParsersRemove := &cobra.Command{ - Use: "remove [config]", + Use: "remove ...", Short: "Remove given parser(s)", - Long: `Remove given parse(s) from hub`, - Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`, + Long: `Remove one or more parsers`, + Example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -165,8 +165,8 @@ func NewParsersRemoveCmd() *cobra.Command { flags := cmdParsersRemove.Flags() flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: Remove tainted and outdated files") - flags.Bool("all", false, "Delete all the parsers") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the parsers") return cmdParsersRemove } @@ -185,7 +185,9 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { } if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + return err + } return nil } @@ -194,7 +196,9 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil { + return err + } } return nil @@ -202,10 +206,10 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { func NewParsersUpgradeCmd() *cobra.Command { cmdParsersUpgrade := &cobra.Command{ - Use: "upgrade [config]", + Use: "upgrade ...", Short: "Upgrade given parser(s)", - Long: `Fetch and upgrade given parser(s) from hub`, - Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more parsers from the hub`, + Example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) @@ -214,8 +218,8 @@ func NewParsersUpgradeCmd() *cobra.Command { } flags := cmdParsersUpgrade.Flags() - flags.Bool("all", false, "Upgrade all the parsers") - flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + flags.BoolP("all", "a", false, "Upgrade all the parsers") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdParsersUpgrade } @@ -232,17 +236,26 @@ func runParsersInspect(cmd *cobra.Command, args []string) error { csConfig.Cscli.PrometheusUrl = url } - InspectItem(args[0], cwhub.PARSERS) + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil { + return err + } + } return nil } func NewParsersInspectCmd() *cobra.Command { cmdParsersInspect := &cobra.Command{ - Use: "inspect [name]", - Short: "Inspect given parser", - Long: `Inspect given parser`, - Example: `cscli parsers inspect crowdsec/xxx`, + Use: "inspect ", + Short: "Inspect a parser", + Long: `Inspect a parser`, + Example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -253,6 +266,7 @@ func NewParsersInspectCmd() *cobra.Command { flags := cmdParsersInspect.Flags() flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdParsersInspect } @@ -273,11 +287,12 @@ func runParsersList(cmd *cobra.Command, args []string) error { func NewParsersListCmd() *cobra.Command { cmdParsersList := &cobra.Command{ - Use: "list [name]", - Short: "List all parsers or given one", - Long: `List all parsers or given one`, + Use: "list [parser... | -a]", + Short: "List parsers", + Long: `List of installed/available/specified parsers`, Example: `cscli parsers list -cscli parser list crowdsecurity/xxx`, +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, DisableAutoGenTag: true, RunE: runParsersList, } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index f94169f56ae..e8926d7cd07 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -13,13 +13,13 @@ import ( func NewPostOverflowsCmd() *cobra.Command { cmdPostOverflows := &cobra.Command{ - Use: "postoverflows [action] [config]", - Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub", - Example: `cscli postoverflows install crowdsecurity/cdn-whitelist -cscli postoverflows inspect crowdsecurity/cdn-whitelist -cscli postoverflows upgrade crowdsecurity/cdn-whitelist -cscli postoverflows list -cscli postoverflows remove crowdsecurity/cdn-whitelist + Use: "postoverflows [postoverflow]...", + Short: "Manage hub postoverflows", + Example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns `, Args: cobra.MinimumNArgs(1), Aliases: []string{"postoverflow"}, @@ -88,10 +88,10 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { func NewPostOverflowsInstallCmd() *cobra.Command { cmdPostOverflowsInstall := &cobra.Command{ - Use: "install [config]", + Use: "install ...", Short: "Install given postoverflow(s)", - Long: `Fetch and install given postoverflow(s) from hub`, - Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more postoverflows from the hub`, + Example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -102,7 +102,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command { flags := cmdPostOverflowsInstall.Flags() flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows") return cmdPostOverflowsInstall @@ -151,10 +151,10 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { func NewPostOverflowsRemoveCmd() *cobra.Command { cmdPostOverflowsRemove := &cobra.Command{ - Use: "remove [config]", + Use: "remove ...", Short: "Remove given postoverflow(s)", - Long: `remove given postoverflow(s)`, - Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`, + Long: `remove one or more postoverflows from the hub`, + Example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -165,7 +165,7 @@ func NewPostOverflowsRemoveCmd() *cobra.Command { flags := cmdPostOverflowsRemove.Flags() flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove : Remove tainted and outdated files") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") flags.Bool("all", false, "Delete all the postoverflows") return cmdPostOverflowsRemove @@ -185,7 +185,9 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil { + return err + } return nil } @@ -194,7 +196,9 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, force); err != nil { + return err + } } return nil @@ -202,10 +206,10 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { func NewPostOverflowsUpgradeCmd() *cobra.Command { cmdPostOverflowsUpgrade := &cobra.Command{ - Use: "upgrade [config]", + Use: "upgrade ...", Short: "Upgrade given postoverflow(s)", - Long: `Fetch and Upgrade given postoverflow(s) from hub`, - Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more postoverflows from the hub`, + Example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) @@ -215,7 +219,7 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command { flags := cmdPostOverflowsUpgrade.Flags() flags.BoolP("all", "a", false, "Upgrade all the postoverflows") - flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdPostOverflowsUpgrade } @@ -232,17 +236,26 @@ func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { csConfig.Cscli.PrometheusUrl = url } - InspectItem(args[0], cwhub.PARSERS_OVFLW) + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.PARSERS_OVFLW, noMetrics); err != nil { + return err + } + } return nil } func NewPostOverflowsInspectCmd() *cobra.Command { cmdPostOverflowsInspect := &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given postoverflow", - Long: `Inspect given postoverflow`, - Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`, + Use: "inspect ", + Short: "Inspect a postoverflow", + Long: `Inspect a postoverflow`, + Example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -254,6 +267,7 @@ func NewPostOverflowsInspectCmd() *cobra.Command { flags := cmdPostOverflowsInspect.Flags() // XXX: is this needed for postoverflows? flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdPostOverflowsInspect } @@ -274,11 +288,12 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error { func NewPostOverflowsListCmd() *cobra.Command { cmdPostOverflowsList := &cobra.Command{ - Use: "list [config]", - Short: "List all postoverflows or given one", - Long: `List all postoverflows or given one`, + Use: "list [postoverflow]...", + Short: "List postoverflows", + Long: `List of installed/available/specified postoverflows`, Example: `cscli postoverflows list -cscli postoverflows list crowdsecurity/xxx`, +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, DisableAutoGenTag: true, RunE: runPostOverflowsList, } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 9a9aa825f26..25044d5f980 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -13,13 +13,13 @@ import ( func NewScenariosCmd() *cobra.Command { cmdScenarios := &cobra.Command{ - Use: "scenarios [action] [config]", - Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub", - Example: `cscli scenarios list [-a] -cscli scenarios install crowdsecurity/ssh-bf -cscli scenarios inspect crowdsecurity/ssh-bf -cscli scenarios upgrade crowdsecurity/ssh-bf -cscli scenarios remove crowdsecurity/ssh-bf + Use: "scenarios [scenario]...", + Short: "Manage hub scenarios", + Example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing `, Args: cobra.MinimumNArgs(1), Aliases: []string{"scenario"}, @@ -88,10 +88,10 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error { func NewCmdScenariosInstall() *cobra.Command { cmdScenariosInstall := &cobra.Command{ - Use: "install [config]", + Use: "install ...", Short: "Install given scenario(s)", - Long: `Fetch and install given scenario(s) from hub`, - Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more scenarios from the hub`, + Example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -102,7 +102,7 @@ func NewCmdScenariosInstall() *cobra.Command { flags := cmdScenariosInstall.Flags() flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios") return cmdScenariosInstall @@ -151,10 +151,10 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error { func NewCmdScenariosRemove() *cobra.Command { cmdScenariosRemove := &cobra.Command{ - Use: "remove [config]", + Use: "remove ...", Short: "Remove given scenario(s)", - Long: `remove given scenario(s)`, - Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`, + Long: `remove one or more scenarios`, + Example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -165,8 +165,8 @@ func NewCmdScenariosRemove() *cobra.Command { flags := cmdScenariosRemove.Flags() flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: Remove tainted and outdated files") - flags.Bool("all", false, "Delete all the scenarios") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the scenarios") return cmdScenariosRemove } @@ -185,7 +185,9 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { } if all { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + return err + } return nil } @@ -194,7 +196,9 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force) + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil { + return err + } } return nil @@ -202,10 +206,10 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { func NewCmdScenariosUpgrade() *cobra.Command { cmdScenariosUpgrade := &cobra.Command{ - Use: "upgrade [config]", + Use: "upgrade ...", Short: "Upgrade given scenario(s)", - Long: `Fetch and Upgrade given scenario(s) from hub`, - Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more scenarios from the hub`, + Example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) @@ -215,7 +219,7 @@ func NewCmdScenariosUpgrade() *cobra.Command { flags := cmdScenariosUpgrade.Flags() flags.BoolP("all", "a", false, "Upgrade all the scenarios") - flags.Bool("force", false, "Force upgrade : Overwrite tainted and outdated files") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdScenariosUpgrade } @@ -232,17 +236,26 @@ func runScenariosInspect(cmd *cobra.Command, args []string) error { csConfig.Cscli.PrometheusUrl = url } - InspectItem(args[0], cwhub.SCENARIOS) + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil { + return err + } + } return nil } func NewCmdScenariosInspect() *cobra.Command { cmdScenariosInspect := &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given scenario", - Long: `Inspect given scenario`, - Example: `cscli scenarios inspect crowdsec/xxx`, + Use: "inspect ", + Short: "Inspect a scenario", + Long: `Inspect a scenario`, + Example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -253,6 +266,7 @@ func NewCmdScenariosInspect() *cobra.Command { flags := cmdScenariosInspect.Flags() flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdScenariosInspect } @@ -273,11 +287,12 @@ func runScenariosList(cmd *cobra.Command, args []string) error { func NewCmdScenariosList() *cobra.Command { cmdScenariosList := &cobra.Command{ - Use: "list [config]", - Short: "List all scenario(s) or given one", - Long: `List all scenario(s) or given one`, + Use: "list [scenario]...", + Short: "List scenarios", + Long: `List of installed/available/specified scenarios`, Example: `cscli scenarios list -cscli scenarios list crowdsecurity/xxx`, +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, DisableAutoGenTag: true, RunE: runScenariosList, } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 15c53d0048d..ed6d9a85848 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -58,10 +58,6 @@ func stripAnsiString(str string) string { func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") - err := csConfig.LoadPrometheus() - if err != nil { - return nil, nil, err - } if csConfig.Cscli.PrometheusUrl == "" { log.Warn("No Prometheus URL configured, metrics will not be collected") @@ -69,7 +65,7 @@ func collectMetrics() ([]byte, []byte, error) { } humanMetrics := bytes.NewBuffer(nil) - err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human") + err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human") if err != nil { return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 82b98cbeff8..27c79f664f3 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -41,9 +41,9 @@ func printHelp(cmd *cobra.Command) { func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { errMsg := "" if score < MaxDistance { - errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem) + errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) } else { - errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem) + errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) } if ignoreErr { log.Error(errMsg) @@ -185,33 +185,40 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, } } -func InspectItem(name string, objecitemType string) { - - hubItem := cwhub.GetItem(objecitemType, name) +func InspectItem(name string, itemType string, noMetrics bool) error { + hubItem := cwhub.GetItem(itemType, name) if hubItem == nil { - log.Fatalf("unable to retrieve item.") + return fmt.Errorf("can't find '%s' in %s", name, itemType) } - var b []byte - var err error + + var ( + b []byte + err error + ) + switch csConfig.Cscli.Output { case "human", "raw": b, err = yaml.Marshal(*hubItem) if err != nil { - log.Fatalf("unable to marshal item : %s", err) + return fmt.Errorf("unable to marshal item: %s", err) } case "json": b, err = json.MarshalIndent(*hubItem, "", " ") if err != nil { - log.Fatalf("unable to marshal item : %s", err) + return fmt.Errorf("unable to marshal item: %s", err) } } + fmt.Printf("%s", string(b)) - if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return + + if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { + return nil } - fmt.Printf("\nCurrent metrics : \n") + fmt.Printf("\nCurrent metrics: \n") ShowMetrics(hubItem) + + return nil } func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 6b0bf5ae4ad..a5ca39769ac 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -1,5 +1,9 @@ package csconfig +import ( + "fmt" +) + /*cscli specific config, such as hub directory*/ type CscliCfg struct { Output string `yaml:"output,omitempty"` @@ -27,5 +31,9 @@ func (c *Config) LoadCSCLI() error { c.Cscli.HubDir = c.ConfigPaths.HubDir c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile + if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { + c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) + } + return nil } diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index b3d0abc6b99..03c3c95abe5 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -38,12 +38,19 @@ func TestLoadCSCLI(t *testing.T) { HubDir: "./hub", HubIndexFile: "./hub/.index.json", }, + Prometheus: &PrometheusCfg{ + Enabled: true, + Level: "full", + ListenAddr: "127.0.0.1", + ListenPort: 6060, + }, }, expected: &CscliCfg{ - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + ConfigDir: configDirFullPath, + DataDir: dataFullPath, + HubDir: hubFullPath, + HubIndexFile: hubIndexFileFullPath, + PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, { diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index f92771bb157..9b80fe39838 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -1,19 +1,8 @@ package csconfig -import "fmt" - type PrometheusCfg struct { Enabled bool `yaml:"enabled"` Level string `yaml:"level"` //aggregated|full ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` } - -func (c *Config) LoadPrometheus() error { - if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil { - if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { - c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) - } - } - return nil -} diff --git a/pkg/csconfig/prometheus_test.go b/pkg/csconfig/prometheus_test.go deleted file mode 100644 index c16002221c6..00000000000 --- a/pkg/csconfig/prometheus_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package csconfig - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadPrometheus(t *testing.T) { - tests := []struct { - name string - input *Config - expectedURL string - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Prometheus: &PrometheusCfg{ - Enabled: true, - Level: "full", - ListenAddr: "127.0.0.1", - ListenPort: 6060, - }, - Cscli: &CscliCfg{}, - }, - expectedURL: "http://127.0.0.1:6060/metrics", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadPrometheus() - cstest.RequireErrorContains(t, err, tc.expectedErr) - - require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl) - }) - } -} diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 8ffeaa541a1..ad9032f8140 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -107,7 +107,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo if name != "" { item := GetItem(itemType, name) if item == nil { - return fmt.Errorf("unable to retrieve: %s", name) + return fmt.Errorf("can't find '%s' in %s", name, itemType) } err := DisableItem(csConfig.Hub, item, purge, forceAction) @@ -151,7 +151,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo return nil } -func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { +func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error { updated := 0 found := false @@ -166,17 +166,17 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if !v.Downloaded { - log.Warningf("%s : not downloaded, please install.", v.Name) + log.Warningf("%s: not downloaded, please install.", v.Name) continue } found = true if v.UpToDate { - log.Infof("%s : up-to-date", v.Name) + log.Infof("%s: up-to-date", v.Name) if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + return fmt.Errorf("%s: download failed: %w", v.Name, err) } if !force { @@ -185,7 +185,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + return fmt.Errorf("%s: download failed: %w", v.Name, err) } if !v.UpToDate { @@ -203,14 +203,14 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) + return fmt.Errorf("unable to add %s: %w", v.Name, err) } } if !found && name == "" { log.Infof("No %s installed, nothing to upgrade", itemType) } else if !found { - log.Errorf("Item '%s' not found in hub", name) + log.Errorf("can't find '%s' in %s", name, itemType) } else if updated == 0 && found { if name == "" { log.Infof("All %s are already up-to-date", itemType) @@ -220,4 +220,6 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } else if updated != 0 { log.Infof("Upgraded %d items", updated) } + + return nil } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 522dead05b0..e7e1314c6c2 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -45,7 +45,8 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) @@ -85,11 +86,12 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - if err := UpdateHubIdx(cfg.Hub); err != nil { + if err = UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) getHubIdxOrFail(t) require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) @@ -141,14 +143,16 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { + if err = UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) getHubIdxOrFail(t) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) + getHubIdxOrFail(t) require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) diff --git a/test/bats/20_collections.bats b/test/bats/20_collections.bats deleted file mode 100644 index aa1fa6b21d0..00000000000 --- a/test/bats/20_collections.bats +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - ./instance-data load - ./instance-crowdsec start -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "we can list collections" { - rune -0 cscli collections list -} - -@test "there are 2 collections (linux and sshd)" { - rune -0 cscli collections list -o json - rune -0 jq '.collections | length' <(output) - assert_output 2 -} - -@test "can install a collection (as a regular user) and remove it" { - # collection is not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections install crowdsecurity/mysql -o human - assert_stderr --partial "Enabled crowdsecurity/mysql" - - # it has been installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - assert_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections remove crowdsecurity/mysql -o human - assert_stderr --partial "Removed symlink [crowdsecurity/mysql]" - - # it has been removed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" -} - -@test "must use --force to remove a collection that belongs to another, which becomes tainted" { - # we expect no error since we may have multiple collections, some removed and some not - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to other collections" - assert_stderr --partial "[crowdsecurity/linux]" - - rune -0 cscli collections remove crowdsecurity/sshd --force - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - rune -0 cscli collections inspect crowdsecurity/linux -o json - rune -0 jq -r '.tainted' <(output) - assert_output "true" -} - -@test "can remove a collection" { - rune -0 cscli collections remove crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." - rune -0 cscli collections inspect crowdsecurity/linux -o human - assert_line 'installed: false' -} - -@test "collections delete is an alias for collections remove" { - rune -0 cscli collections delete crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "removing a collection that does not exist is noop" { - rune -0 cscli collections remove crowdsecurity/apache2 - refute_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "can remove a removed collection" { - rune -0 cscli collections install crowdsecurity/mysql - rune -0 cscli collections remove crowdsecurity/mysql - assert_stderr --partial "Removed" - rune -0 cscli collections remove crowdsecurity/mysql - refute_stderr --partial "Removed" -} - -@test "can remove all collections" { - # we may have this too, from package installs - rune cscli parsers delete crowdsecurity/whitelists - rune -0 cscli collections remove --all - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - assert_stderr --partial "Removed symlink [crowdsecurity/linux]" - rune -0 cscli hub list -o json - assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' - rune -0 cscli collections remove --all - assert_stderr --partial 'Disabled 0 items' -} - -@test "a taint bubbles up to the top collection" { - coll=crowdsecurity/nginx - subcoll=crowdsecurity/base-http-scenarios - scenario=crowdsecurity/http-crawl-non_statics - - # install a collection with dependencies - rune -0 cscli collections install "$coll" - - # the collection, subcollection and scenario are installed and not tainted - # we have to default to false because tainted is (as of 1.4.6) returned - # only when true - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - - # we taint the scenario - HUB_DIR=$(config_get '.config_paths.hub_dir') - yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" - - # the collection, subcollection and scenario are now tainted - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -} - -# TODO test download-only diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats new file mode 100644 index 00000000000..db33ae97fb2 --- /dev/null +++ b/test/bats/20_hub_collections.bats @@ -0,0 +1,319 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections list" { + # no items + rune -0 cscli collections list + assert_output --partial "COLLECTIONS" + rune -0 cscli collections list -o json + assert_json '{collections:[]}' + rune -0 cscli collections list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + rune -0 cscli collections list + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli collections list -o json + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 jq '.collections | length' <(output) + assert_output "2" + + rune -0 cscli collections list -o raw + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.collections | length') + + rune -0 cscli collections list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli collections list -o json -a + rune -0 jq '.collections | length' <(output) + assert_output "$expected" + + rune -0 cscli collections list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli collections list [collection]..." { + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # list one item + rune -0 cscli collections list crowdsecurity/sshd + assert_output --partial "crowdsecurity/sshd" + refute_output --partial "crowdsecurity/smb" + + # list multiple items + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb + assert_output --partial "crowdsecurity/sshd" + assert_output --partial "crowdsecurity/smb" + + rune -0 cscli collections list crowdsecurity/sshd -o json + rune -0 jq '.collections | length' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 jq '.collections | length' <(output) + assert_output "2" + + rune -0 cscli collections list crowdsecurity/sshd -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections list [collection]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli collections list crowdsecurity/sshd + # not existing + rune -1 cscli collections list blahblah/blahblah +} + +@test "cscli collections install [collection]..." { + rune -1 cscli collections install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli collections install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in collections" + + # simple install + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli collections install crowdsecurity/ssshd + assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean crowdsecurity/sshd?" + + # install multiple + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + rune -0 cscli collections inspect crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/smb' + assert_output --partial 'installed: true' +} + +@test "cscli collections install [collection]... (file location and download-only)" { + # simple install + rune -0 cscli collections install crowdsecurity/linux --download-only + rune -0 cscli collections inspect crowdsecurity/linux --no-metrics + assert_output --partial 'crowdsecurity/linux' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/collections/crowdsecurity/linux.yaml" + assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml" + + rune -0 cscli collections install crowdsecurity/linux + assert_file_exists "$CONFIG_DIR/collections/linux.yaml" +} + + +@test "cscli collections inspect [collection]..." { + rune -1 cscli collections inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli collections inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # one item + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null]' + + # one item, raw + rune -0 cscli collections inspect crowdsecurity/sshd -o raw + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",null]]' + + # multiple items, raw + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli collections remove [collection]..." { + rune -1 cscli collections remove + assert_stderr --partial "specify at least one collection to remove or '--all'" + + rune -1 cscli collections remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli collections remove crowdsecurity/sshd + + # install, then remove, check files + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # delete is an alias for remove + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections delete crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # purge + assert_file_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_file_not_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # --all + rune -0 cscli collections list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli collections remove --all + + rune -0 cscli collections list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli collections upgrade [collection]..." { + rune -1 cscli collections upgrade + assert_stderr --partial "specify at least one collection to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli collections upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli collections upgrade crowdsecurity/sshd + assert_stderr --partial "can't find 'crowdsecurity/sshd' in collections" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/sshd":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli collections install crowdsecurity/sshd + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli collections upgrade crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + # XXX: should return error + rune -0 cscli collections upgrade crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd is tainted, --force to overwrite" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli collections upgrade crowdsecurity/sshd --force + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli collections install crowdsecurity/smb + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].local_version]==["?","?"]' <(output) + rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb + rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections upgrade --all + rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) +} diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats new file mode 100644 index 00000000000..26844c96586 --- /dev/null +++ b/test/bats/20_hub_collections_dep.bats @@ -0,0 +1,86 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections (dependencies)" { + # inject a dependency: smb requires sshd + hub_dep=$(jq <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}') + echo "$hub_dep" >"$HUB_DIR/.index.json" + + # verify that installing smb brings sshd + rune -0 cscli collections install crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].name]==["crowdsecurity/smb","crowdsecurity/sshd"]' <(output) + + # verify that removing smb removes sshd too + rune -0 cscli collections remove crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # we can't remove sshd without --force + rune -0 cscli collections install crowdsecurity/smb + # XXX: should this be an error? + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection" + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]' + + # use the --force + rune -0 cscli collections remove crowdsecurity/sshd --force + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb"]' + + # and now smb is tainted! + rune -0 cscli collections inspect crowdsecurity/smb -o json + rune -0 jq -e '.tainted//false==true' <(output) + rune -0 cscli collections remove crowdsecurity/smb --force + + # empty + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # reinstall + rune -0 cscli collections install crowdsecurity/smb --force + + # taint on sshd means smb is tainted as well + rune -0 cscli collections inspect crowdsecurity/smb -o json + jq -e '.tainted//false==false' <(output) + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/smb -o json + jq -e '.tainted//false==true' <(output) + + # now we can't remove smb without --force + rune -1 cscli collections remove crowdsecurity/smb + assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite" +} diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats new file mode 100644 index 00000000000..1840d14ec2f --- /dev/null +++ b/test/bats/20_hub_parsers.bats @@ -0,0 +1,416 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + # XXX: remove all "content" fields from the index, to make sure + # XXX: we don't rely on it in any way + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers list" { + # no items + rune -0 cscli parsers list + assert_output --partial "PARSERS" + rune -0 cscli parsers list -o json + assert_json '{parsers:[]}' + rune -0 cscli parsers list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + rune -0 cscli parsers list + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli parsers list -o json + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 jq '.parsers | length' <(output) + assert_output "2" + + rune -0 cscli parsers list -o raw + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli parsers list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.parsers | length') + + rune -0 cscli parsers list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o json -a + rune -0 jq '.parsers | length' <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli parsers list [parser]..." { + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # list one item + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --partial "crowdsecurity/whitelists" + refute_output --partial "crowdsecurity/windows-auth" + + # list multiple items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth + assert_output --partial "crowdsecurity/whitelists" + assert_output --partial "crowdsecurity/windows-auth" + + rune -0 cscli parsers list crowdsecurity/whitelists -o json + rune -0 jq '.parsers | length' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o json + rune -0 jq '.parsers | length' <(output) + assert_output "2" + + rune -0 cscli parsers list crowdsecurity/whitelists -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli parsers list [parser]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli parsers list crowdsecurity/whitelists + # not existing + rune -1 cscli parsers list blahblah/blahblah +} + +@test "cscli parsers install [parser]..." { + rune -1 cscli parsers install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli parsers install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in parsers" + + # simple install + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli parsers install crowdsecurity/sshd-logz + assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean crowdsecurity/sshd-logs?" + + # install multiple + rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs + rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics + assert_output --partial 'crowdsecurity/pgsql-logs' + assert_output --partial 'installed: true' + rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics + assert_output --partial 'crowdsecurity/postfix-logs' + assert_output --partial 'installed: true' +} + +@test "cscli parsers install [parser]... (file location and download-only)" { + # simple install + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +# XXX: test install with --force +# XXX: test install with --ignore + +@test "cscli parsers inspect [parser]..." { + rune -1 cscli parsers inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli parsers inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # one item + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics + assert_line 'type: parsers' + assert_line 'stage: s01-parse' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null]' + + # one item, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw + assert_line 'type: parsers' + assert_line 'stage: s01-parse' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",null]]' + + # multiple items, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli parsers remove [parser]..." { + rune -1 cscli parsers remove + assert_stderr --partial "specify at least one parser to remove or '--all'" + + rune -1 cscli parsers remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli parsers remove crowdsecurity/whitelists + + # XXX: have the --force ignore uninstalled items + # XXX: maybe also with --purge + + # install, then remove, check files + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # delete is an alias for remove + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers delete crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # purge + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # --all + rune -0 cscli parsers list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli parsers remove --all + + rune -0 cscli parsers list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli parsers upgrade [parser]..." { + rune -1 cscli parsers upgrade + assert_stderr --partial "specify at least one parser to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli parsers upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_stderr --partial "can't find 'crowdsecurity/whitelists' in parsers" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {parsers:{"crowdsecurity/whitelists":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli parsers install crowdsecurity/whitelists + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli parsers upgrade crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + # XXX: should return error + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_stderr --partial "crowdsecurity/whitelists is tainted, --force to overwrite" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli parsers upgrade crowdsecurity/whitelists --force + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli parsers install crowdsecurity/windows-auth + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '[.parsers[].local_version]==["?","?"]' <(output) + rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth + rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers upgrade --all + rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) +} + + + +#@test "must use --force to remove a collection that belongs to another, which becomes tainted" { +# # we expect no error since we may have multiple collections, some removed and some not +# rune -0 cscli collections remove crowdsecurity/sshd +# assert_stderr --partial "crowdsecurity/sshd belongs to other collections" +# assert_stderr --partial "[crowdsecurity/linux]" +# +# rune -0 cscli collections remove crowdsecurity/sshd --force +# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" +# rune -0 cscli collections inspect crowdsecurity/linux -o json +# rune -0 jq -r '.tainted' <(output) +# assert_output "true" +#} +# +#@test "can remove a collection" { +# rune -0 cscli collections remove crowdsecurity/linux +# assert_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +# rune -0 cscli collections inspect crowdsecurity/linux -o human --no-metrics +# assert_line 'installed: false' +#} +# +#@test "collections delete is an alias for collections remove" { +# rune -0 cscli collections delete crowdsecurity/linux +# assert_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +#} +# +#@test "removing a collection that does not exist is noop" { +# rune -0 cscli collections remove crowdsecurity/apache2 +# refute_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +#} +# +#@test "can remove a removed collection" { +# rune -0 cscli collections install crowdsecurity/mysql +# rune -0 cscli collections remove crowdsecurity/mysql +# assert_stderr --partial "Removed" +# rune -0 cscli collections remove crowdsecurity/mysql +# refute_stderr --partial "Removed" +#} +# +#@test "can remove all collections" { +# # we may have this too, from package installs +# rune cscli parsers delete crowdsecurity/whitelists +# rune -0 cscli collections remove --all +# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" +# assert_stderr --partial "Removed symlink [crowdsecurity/linux]" +# rune -0 cscli hub list -o json +# assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' +# rune -0 cscli collections remove --all +# assert_stderr --partial 'Disabled 0 items' +#} +# +#@test "a taint bubbles up to the top collection" { +# coll=crowdsecurity/nginx +# subcoll=crowdsecurity/base-http-scenarios +# scenario=crowdsecurity/http-crawl-non_statics +# +# # install a collection with dependencies +# rune -0 cscli collections install "$coll" +# +# # the collection, subcollection and scenario are installed and not tainted +# # we have to default to false because tainted is (as of 1.4.6) returned +# # only when true +# rune -0 cscli collections inspect "$coll" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# rune -0 cscli collections inspect "$subcoll" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# rune -0 cscli scenarios inspect "$scenario" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# +# # we taint the scenario +# HUB_DIR=$(config_get '.config_paths.hub_dir') +# yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" +# +# # the collection, subcollection and scenario are now tainted +# rune -0 cscli scenarios inspect "$scenario" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +# rune -0 cscli collections inspect "$subcoll" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +# rune -0 cscli collections inspect "$coll" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +#} +# +## TODO test download-only diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats new file mode 100644 index 00000000000..cf74268f724 --- /dev/null +++ b/test/bats/20_hub_postoverflows.bats @@ -0,0 +1,321 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli postoverflows list" { + # no items + rune -0 cscli postoverflows list + assert_output --partial "POSTOVERFLOWS" + rune -0 cscli postoverflows list -o json + assert_json '{postoverflows:[]}' + rune -0 cscli postoverflows list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + rune -0 cscli postoverflows list + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o json + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 jq '.postoverflows | length' <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o raw + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli postoverflows list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.postoverflows | length') + + rune -0 cscli postoverflows list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o json -a + rune -0 jq '.postoverflows | length' <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli postoverflows list [scenario]..." { + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # list one item + rune -0 cscli postoverflows list crowdsecurity/rdns + assert_output --partial "crowdsecurity/rdns" + refute_output --partial "crowdsecurity/cdn-whitelist" + + # list multiple items + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist + assert_output --partial "crowdsecurity/rdns" + assert_output --partial "crowdsecurity/cdn-whitelist" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "2" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli postoverflows list [scenario]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli postoverflows list crowdsecurity/rdns + # not existing + rune -1 cscli postoverflows list blahblah/blahblah +} + +@test "cscli postoverflows install [scenario]..." { + rune -1 cscli postoverflows install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli postoverflows install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in postoverflows" + + # simple install + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli postoverflows install crowdsecurity/rdnf + assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean crowdsecurity/rdns?" + + # install multiple + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + rune -0 cscli postoverflows inspect crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/cdn-whitelist' + assert_output --partial 'installed: true' +} + +@test "cscli postoverflows install [postoverflow]... (file location and download-only)" { + # simple install + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" +} + + +@test "cscli postoverflows inspect [scenario]..." { + rune -1 cscli postoverflows inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli postoverflows inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # one item + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_line 'type: postoverflows' + assert_line 'stage: s00-enrich' + assert_line 'name: crowdsecurity/rdns' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null]' + + # one item, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw + assert_line 'type: postoverflows' + assert_line 'stage: s00-enrich' + assert_line 'name: crowdsecurity/rdns' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",null]]' + + # multiple items, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli postoverflows remove [postoverflow]..." { + rune -1 cscli postoverflows remove + assert_stderr --partial "specify at least one postoverflow to remove or '--all'" + + rune -1 cscli postoverflows remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli postoverflows remove crowdsecurity/rdns + + # install, then remove, check files + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # delete is an alias for remove + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows delete crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # purge + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_file_not_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # --all + rune -0 cscli postoverflows list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli postoverflows remove --all + + rune -0 cscli postoverflows list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli postoverflows upgrade [postoverflow]..." { + rune -1 cscli postoverflows upgrade + assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli postoverflows upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + assert_stderr --partial "can't find 'crowdsecurity/rdns' in postoverflows" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {postoverflows:{"crowdsecurity/rdns":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli postoverflows install crowdsecurity/rdns + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + # XXX: should return error + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + assert_stderr --partial "crowdsecurity/rdns is tainted, --force to overwrite" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli postoverflows upgrade crowdsecurity/rdns --force + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows list -o json + rune -0 jq -e '[.postoverflows[].local_version]==["?","?"]' <(output) + rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows upgrade --all + rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) +} diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats new file mode 100644 index 00000000000..ecec032d178 --- /dev/null +++ b/test/bats/20_hub_scenarios.bats @@ -0,0 +1,320 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli scenarios list" { + # no items + rune -0 cscli scenarios list + assert_output --partial "SCENARIOS" + rune -0 cscli scenarios list -o json + assert_json '{scenarios:[]}' + rune -0 cscli scenarios list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + rune -0 cscli scenarios list + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli scenarios list -o json + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 jq '.scenarios | length' <(output) + assert_output "2" + + rune -0 cscli scenarios list -o raw + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli scenarios list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.scenarios | length') + + rune -0 cscli scenarios list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o json -a + rune -0 jq '.scenarios | length' <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli scenarios list [scenario]..." { + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # list one item + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --partial "crowdsecurity/ssh-bf" + refute_output --partial "crowdsecurity/telnet-bf" + + # list multiple items + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf + assert_output --partial "crowdsecurity/ssh-bf" + assert_output --partial "crowdsecurity/telnet-bf" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "2" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli scenarios list [scenario]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli scenarios list crowdsecurity/ssh-bf + # not existing + rune -1 cscli scenarios list blahblah/blahblah +} + +@test "cscli scenarios install [scenario]..." { + rune -1 cscli scenarios install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli scenarios install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in scenarios" + + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli scenarios install crowdsecurity/ssh-tf + assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean crowdsecurity/ssh-bf?" + + # install multiple + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + rune -0 cscli scenarios inspect crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/telnet-bf' + assert_output --partial 'installed: true' +} + + +@test "cscli scenarios install [scenario]... (file location and download-only)" { + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" +} + + +@test "cscli scenarios inspect [scenario]..." { + rune -1 cscli scenarios inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli scenarios inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # one item + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null]' + + # one item, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",null]]' + + # multiple items, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli scenarios remove [scenario]..." { + rune -1 cscli scenarios remove + assert_stderr --partial "specify at least one scenario to remove or '--all'" + + rune -1 cscli scenarios remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + + # install, then remove, check files + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # delete is an alias for remove + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios delete crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # purge + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_file_not_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # --all + rune -0 cscli scenarios list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli scenarios remove --all + + rune -0 cscli scenarios list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli scenarios upgrade [scenario]..." { + rune -1 cscli scenarios upgrade + assert_stderr --partial "specify at least one scenario to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli scenarios upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + assert_stderr --partial "can't find 'crowdsecurity/ssh-bf' in scenarios" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {scenarios:{"crowdsecurity/ssh-bf":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + # XXX: should return error + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf is tainted, --force to overwrite" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf --force + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli scenarios install crowdsecurity/telnet-bf + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios list -o json + rune -0 jq -e '[.scenarios[].local_version]==["?","?"]' <(output) + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios upgrade --all + rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) +} diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 5e16340ec9d..bcd216091f1 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -238,6 +238,12 @@ assert_stderr_line() { } export -f assert_stderr_line +hub_uninstall_all() { + CONFIG_DIR=$(dirname "$CONFIG_YAML") + rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/* +} +export -f hub_uninstall_all + # remove color and style sequences from stdin plaintext() { sed -E 's/\x1B\[[0-9;]*[JKmsu]//g' From 325003bb690a58571041233fa531fcea18b4e717 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:33:36 +0200 Subject: [PATCH 095/263] Refact cscli item listing, tests (#2547) * hub diet; taint tests * cmd/crowdsec-cli: split utils.go, moved cwhub.GetHubStatusForItemType() * cscli: refactor hub list commands, fix edge cases --- cmd/crowdsec-cli/collections.go | 10 +- cmd/crowdsec-cli/hub.go | 9 +- cmd/crowdsec-cli/item_metrics.go | 245 +++++++++++++++++ cmd/crowdsec-cli/item_suggest.go | 93 +++++++ cmd/crowdsec-cli/items.go | 174 ++++++++++++ cmd/crowdsec-cli/parsers.go | 5 +- cmd/crowdsec-cli/postoverflows.go | 5 +- cmd/crowdsec-cli/scenarios.go | 5 +- cmd/crowdsec-cli/support.go | 4 +- cmd/crowdsec-cli/utils.go | 429 ------------------------------ cmd/crowdsec-cli/utils_table.go | 8 +- pkg/cwhub/cwhub.go | 82 ++---- pkg/cwhub/cwhub_test.go | 11 +- test/bats/20_hub_parsers.bats | 35 +-- test/bats/20_hub_scenarios.bats | 35 +-- 15 files changed, 608 insertions(+), 542 deletions(-) create mode 100644 cmd/crowdsec-cli/item_metrics.go create mode 100644 cmd/crowdsec-cli/item_suggest.go create mode 100644 cmd/crowdsec-cli/items.go diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index c7a3e2c56fd..8dbb41fcaf7 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -254,6 +254,11 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error { return err } + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + for _, name := range args { if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { return err @@ -292,8 +297,9 @@ func runCollectionsList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing collections - ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 7b37ec4c7ad..bd0f6f77b2b 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -50,7 +50,7 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + if err = require.Hub(csConfig); err != nil { return err } @@ -62,9 +62,12 @@ func runHubList(cmd *cobra.Command, args []string) error { cwhub.DisplaySummary() - ListItems(color.Output, []string{ + err = ListItems(color.Output, []string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, - }, args, true, false, all) + }, nil, true, false, all) + if err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go new file mode 100644 index 00000000000..1eb8e39e871 --- /dev/null +++ b/cmd/crowdsec-cli/item_metrics.go @@ -0,0 +1,245 @@ +package main + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/prom2json" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/trace" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func ShowMetrics(hubItem *cwhub.Item) { + switch hubItem.Type { + case cwhub.PARSERS: + metrics := GetParserMetric(prometheusURL, hubItem.Name) + parserMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.SCENARIOS: + metrics := GetScenarioMetric(prometheusURL, hubItem.Name) + scenarioMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.COLLECTIONS: + for _, item := range hubItem.Parsers { + metrics := GetParserMetric(prometheusURL, item) + parserMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Scenarios { + metrics := GetScenarioMetric(prometheusURL, item) + scenarioMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Collections { + hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) + if hubItem == nil { + log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) + } + ShowMetrics(hubItem) + } + default: + log.Errorf("item of type '%s' is unknown", hubItem.Type) + } +} + +// GetParserMetric is a complete rip from prom2json +func GetParserMetric(url string, itemName string) map[string]map[string]int { + stats := make(map[string]map[string]int) + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + source, ok := metric.Labels["source"] + if !ok { + log.Debugf("no source in Metric %v", metric.Labels) + } else { + if srctype, ok := metric.Labels["type"]; ok { + source = srctype + ":" + source + } + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_reader_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + stats[source]["parsed"] = 0 + stats[source]["reads"] = 0 + stats[source]["unparsed"] = 0 + stats[source]["hits"] = 0 + } + stats[source]["reads"] += ival + case "cs_parser_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_parser_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + case "cs_node_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["hits"] += ival + case "cs_node_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_node_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + default: + continue + } + } + } + return stats +} + +func GetScenarioMetric(url string, itemName string) map[string]int { + stats := make(map[string]int) + + stats["instantiation"] = 0 + stats["curr_count"] = 0 + stats["overflow"] = 0 + stats["pour"] = 0 + stats["underflow"] = 0 + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_bucket_created_total": + stats["instantiation"] += ival + case "cs_buckets": + stats["curr_count"] += ival + case "cs_bucket_overflowed_total": + stats["overflow"] += ival + case "cs_bucket_poured_total": + stats["pour"] += ival + case "cs_bucket_underflowed_total": + stats["underflow"] += ival + default: + continue + } + } + } + return stats +} + +func GetPrometheusMetric(url string) []*prom2json.Family { + mfChan := make(chan *dto.MetricFamily, 1024) + + // Start with the DefaultTransport for sane defaults. + transport := http.DefaultTransport.(*http.Transport).Clone() + // Conservatively disable HTTP keep-alives as this program will only + // ever need a single HTTP request. + transport.DisableKeepAlives = true + // Timeout early if the server doesn't even return the headers. + transport.ResponseHeaderTimeout = time.Minute + + go func() { + defer trace.CatchPanic("crowdsec/GetPrometheusMetric") + err := prom2json.FetchMetricFamilies(url, mfChan, transport) + if err != nil { + log.Fatalf("failed to fetch prometheus metrics : %v", err) + } + }() + + result := []*prom2json.Family{} + for mf := range mfChan { + result = append(result, prom2json.NewFamily(mf)) + } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) + + return result +} + +type unit struct { + value int64 + symbol string +} + +var ranges = []unit{ + {value: 1e18, symbol: "E"}, + {value: 1e15, symbol: "P"}, + {value: 1e12, symbol: "T"}, + {value: 1e9, symbol: "G"}, + {value: 1e6, symbol: "M"}, + {value: 1e3, symbol: "k"}, + {value: 1, symbol: ""}, +} + +func formatNumber(num int) string { + goodUnit := unit{} + for _, u := range ranges { + if int64(num) >= u.value { + goodUnit = u + break + } + } + + if goodUnit.value == 1 { + return fmt.Sprintf("%d%s", num, goodUnit.symbol) + } + + res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 + return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) +} diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go new file mode 100644 index 00000000000..c52239fb478 --- /dev/null +++ b/cmd/crowdsec-cli/item_suggest.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/agext/levenshtein" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "slices" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +const MaxDistance = 7 + +func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { + errMsg := "" + if score < MaxDistance { + errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) + } else { + errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) + } + if ignoreErr { + log.Error(errMsg) + } else { + log.Fatalf(errMsg) + } +} + +func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { + allItems := make([]string, 0) + nearestScore := 100 + nearestItem := &cwhub.Item{} + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + allItems = append(allItems, item.Name) + } + + for _, s := range allItems { + d := levenshtein.Distance(itemName, s, nil) + if d < nearestScore { + nearestScore = d + nearestItem = cwhub.GetItem(itemType, s) + } + } + return nearestItem, nearestScore +} + +func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) + } + } + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + return comp, cobra.ShellCompDirectiveNoFileComp +} + +func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + items, err := cwhub.GetInstalledItemsAsString(itemType) + if err != nil { + cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + if toComplete != "" { + for _, item := range items { + if strings.Contains(item, toComplete) { + comp = append(comp, item) + } + } + } else { + comp = items + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go new file mode 100644 index 00000000000..cebf5b723d6 --- /dev/null +++ b/cmd/crowdsec-cli/items.go @@ -0,0 +1,174 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "slices" + "sort" + "strings" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + + +func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) { + itemNames := cwhub.GetItemNames(itemType) + + notExist := []string{} + if len(args) > 0 { + installedOnly = false + for _, arg := range args { + if !slices.Contains(itemNames, arg) { + notExist = append(notExist, arg) + } + } + } + + if len(notExist) > 0 { + return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType) + } + + if len(args) > 0 { + itemNames = args + } + + if installedOnly { + installed := []string{} + for _, item := range itemNames { + if cwhub.GetItem(itemType, item).Installed { + installed = append(installed, item) + } + } + return installed, nil + } + return itemNames, nil +} + + +func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { + var err error + items := make(map[string][]string) + for _, itemType := range itemTypes { + if items[itemType], err = selectItems(itemType, args, !all); err != nil { + return err + } + } + + if csConfig.Cscli.Output == "human" { + for _, itemType := range itemTypes { + listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) + } + } else if csConfig.Cscli.Output == "json" { + type itemHubStatus struct { + Name string `json:"name"` + LocalVersion string `json:"local_version"` + LocalPath string `json:"local_path"` + Description string `json:"description"` + UTF8Status string `json:"utf8_status"` + Status string `json:"status"` + } + + hubStatus := make(map[string][]itemHubStatus) + for _, itemType := range itemTypes { + // empty slice in case there are no items of this type + hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) + for i, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, emo := item.Status() + hubStatus[itemType][i] = itemHubStatus{ + Name: item.Name, + LocalVersion: item.LocalVersion, + LocalPath: item.LocalPath, + Description: item.Description, + Status: status, + UTF8Status: fmt.Sprintf("%v %s", emo, status), + } + } + h := hubStatus[itemType] + sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name }) + } + x, err := json.MarshalIndent(hubStatus, "", " ") + if err != nil { + log.Fatalf("failed to unmarshal") + } + out.Write(x) + } else if csConfig.Cscli.Output == "raw" { + csvwriter := csv.NewWriter(out) + if showHeader { + header := []string{"name", "status", "version", "description"} + if showType { + header = append(header, "type") + } + err := csvwriter.Write(header) + if err != nil { + log.Fatalf("failed to write header: %s", err) + } + + } + for _, itemType := range itemTypes { + for _, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, _ := item.Status() + if item.LocalVersion == "" { + item.LocalVersion = "n/a" + } + row := []string{ + item.Name, + status, + item.LocalVersion, + item.Description, + } + if showType { + row = append(row, itemType) + } + err := csvwriter.Write(row) + if err != nil { + log.Fatalf("failed to write raw output : %s", err) + } + } + } + csvwriter.Flush() + } + return nil +} + +func InspectItem(name string, itemType string, noMetrics bool) error { + hubItem := cwhub.GetItem(itemType, name) + if hubItem == nil { + return fmt.Errorf("can't find '%s' in %s", name, itemType) + } + + var ( + b []byte + err error + ) + + switch csConfig.Cscli.Output { + case "human", "raw": + b, err = yaml.Marshal(*hubItem) + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + case "json": + b, err = json.MarshalIndent(*hubItem, "", " ") + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + } + + fmt.Printf("%s", string(b)) + + if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { + return nil + } + + fmt.Printf("\nCurrent metrics: \n") + ShowMetrics(hubItem) + + return nil +} diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index aee11abd0b9..49e301a6c7c 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -279,8 +279,9 @@ func runParsersList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing parsers - ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index e8926d7cd07..7ff245a9f92 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -280,8 +280,9 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing postoverflows - ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 25044d5f980..863e50236ba 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -279,8 +279,9 @@ func runScenariosList(cmd *cobra.Command, args []string) error { return err } - // XXX: will happily ignore missing scenarios - ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) + if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil { + return err + } return nil } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index ed6d9a85848..6477383721d 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -131,7 +131,9 @@ func collectOSInfo() ([]byte, error) { func collectHubItems(itemType string) []byte { out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - ListItems(out, []string{itemType}, []string{}, false, true, false) + if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } return out.Bytes() } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 27c79f664f3..eb7fb51e02e 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -1,36 +1,17 @@ package main import ( - "encoding/csv" - "encoding/json" "fmt" - "io" - "math" "net" - "net/http" - "slices" - "strconv" "strings" - "time" - "github.com/fatih/color" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/agext/levenshtein" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/trace" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" ) -const MaxDistance = 7 - func printHelp(cmd *cobra.Command) { err := cmd.Help() if err != nil { @@ -38,189 +19,6 @@ func printHelp(cmd *cobra.Command) { } } -func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { - errMsg := "" - if score < MaxDistance { - errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) - } else { - errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) - } - if ignoreErr { - log.Error(errMsg) - } else { - log.Fatalf(errMsg) - } -} - -func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { - allItems := make([]string, 0) - nearestScore := 100 - nearestItem := &cwhub.Item{} - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - allItems = append(allItems, item.Name) - } - - for _, s := range allItems { - d := levenshtein.Distance(itemName, s, nil) - if d < nearestScore { - nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) - } - } - return nearestItem, nearestScore -} - -func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - items, err := cwhub.GetInstalledItemsAsString(itemType) - if err != nil { - cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - - if toComplete != "" { - for _, item := range items { - if strings.Contains(item, toComplete) { - comp = append(comp, item) - } - } - } else { - comp = items - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) { - var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) - - for _, itemType := range itemTypes { - itemName := "" - if len(args) == 1 { - itemName = args[0] - } - hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) - } - - if csConfig.Cscli.Output == "human" { - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses) - } - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(hubStatusByItemType, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - out.Write(x) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(out) - if showHeader { - header := []string{"name", "status", "version", "description"} - if showType { - header = append(header, "type") - } - err := csvwriter.Write(header) - if err != nil { - log.Fatalf("failed to write header: %s", err) - } - - } - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - for _, status := range statuses { - if status.LocalVersion == "" { - status.LocalVersion = "n/a" - } - row := []string{ - status.Name, - status.Status, - status.LocalVersion, - status.Description, - } - if showType { - row = append(row, itemType) - } - err := csvwriter.Write(row) - if err != nil { - log.Fatalf("failed to write raw output : %s", err) - } - } - } - csvwriter.Flush() - } -} - -func InspectItem(name string, itemType string, noMetrics bool) error { - hubItem := cwhub.GetItem(itemType, name) - if hubItem == nil { - return fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - var ( - b []byte - err error - ) - - switch csConfig.Cscli.Output { - case "human", "raw": - b, err = yaml.Marshal(*hubItem) - if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) - } - case "json": - b, err = json.MarshalIndent(*hubItem, "", " ") - if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) - } - } - - fmt.Printf("%s", string(b)) - - if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return nil - } - - fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hubItem) - - return nil -} - func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { /*if a range is provided, change the scope*/ @@ -251,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * return nil } -func ShowMetrics(hubItem *cwhub.Item) { - switch hubItem.Type { - case cwhub.PARSERS: - metrics := GetParserMetric(hubItem.Name) - parserMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.SCENARIOS: - metrics := GetScenarioMetric(hubItem.Name) - scenarioMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.COLLECTIONS: - for _, item := range hubItem.Parsers { - metrics := GetParserMetric(item) - parserMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(item) - scenarioMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) - if hubItem == nil { - log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) - } - ShowMetrics(hubItem) - } - default: - log.Errorf("item of type '%s' is unknown", hubItem.Type) - } -} - -// GetParserMetric is a complete rip from prom2json -func GetParserMetric(itemName string) map[string]map[string]int { - stats := make(map[string]map[string]int) - - result := GetPrometheusMetric() - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - source, ok := metric.Labels["source"] - if !ok { - log.Debugf("no source in Metric %v", metric.Labels) - } else { - if srctype, ok := metric.Labels["type"]; ok { - source = srctype + ":" + source - } - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_reader_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - stats[source]["parsed"] = 0 - stats[source]["reads"] = 0 - stats[source]["unparsed"] = 0 - stats[source]["hits"] = 0 - } - stats[source]["reads"] += ival - case "cs_parser_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_parser_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - case "cs_node_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["hits"] += ival - case "cs_node_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_node_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - default: - continue - } - } - } - return stats -} - -func GetScenarioMetric(itemName string) map[string]int { - stats := make(map[string]int) - - stats["instantiation"] = 0 - stats["curr_count"] = 0 - stats["overflow"] = 0 - stats["pour"] = 0 - stats["underflow"] = 0 - - result := GetPrometheusMetric() - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_bucket_created_total": - stats["instantiation"] += ival - case "cs_buckets": - stats["curr_count"] += ival - case "cs_bucket_overflowed_total": - stats["overflow"] += ival - case "cs_bucket_poured_total": - stats["pour"] += ival - case "cs_bucket_underflowed_total": - stats["underflow"] += ival - default: - continue - } - } - } - return stats -} - -func GetPrometheusMetric() []*prom2json.Family { - mfChan := make(chan *dto.MetricFamily, 1024) - - // Start with the DefaultTransport for sane defaults. - transport := http.DefaultTransport.(*http.Transport).Clone() - // Conservatively disable HTTP keep-alives as this program will only - // ever need a single HTTP request. - transport.DisableKeepAlives = true - // Timeout early if the server doesn't even return the headers. - transport.ResponseHeaderTimeout = time.Minute - - go func() { - defer trace.CatchPanic("crowdsec/GetPrometheusMetric") - err := prom2json.FetchMetricFamilies(csConfig.Cscli.PrometheusUrl, mfChan, transport) - if err != nil { - log.Fatalf("failed to fetch prometheus metrics : %v", err) - } - }() - - result := []*prom2json.Family{} - for mf := range mfChan { - result = append(result, prom2json.NewFamily(mf)) - } - log.Debugf("Finished reading prometheus output, %d entries", len(result)) - - return result -} - -type unit struct { - value int64 - symbol string -} - -var ranges = []unit{ - {value: 1e18, symbol: "E"}, - {value: 1e15, symbol: "P"}, - {value: 1e12, symbol: "T"}, - {value: 1e9, symbol: "G"}, - {value: 1e6, symbol: "M"}, - {value: 1e3, symbol: "k"}, - {value: 1, symbol: ""}, -} - -func formatNumber(num int) string { - goodUnit := unit{} - for _, u := range ranges { - if int64(num) >= u.value { - goodUnit = u - break - } - } - - if goodUnit.value == 1 { - return fmt.Sprintf("%d%s", num, goodUnit.symbol) - } - - res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 - return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) -} - func getDBClient() (*database.Client, error) { var err error if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { @@ -510,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string { } return slice - } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 16f42d72a18..840250f9c8a 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -10,14 +10,16 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) { +func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - for _, status := range statuses { - t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) + for itemName := range itemNames { + item := cwhub.GetItem(itemType, itemNames[itemName]) + status, emo := item.Status() + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 2fe45bcda69..bf4d69d6dcc 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -1,10 +1,13 @@ +// Package cwhub is responsible for installing and upgrading the local hub files. +// +// This includes retrieving the index, the items to install (parsers, scenarios, data files...) +// and managing the dependencies and taints. package cwhub import ( "fmt" "os" "path/filepath" - "sort" "strings" "github.com/enescakir/emoji" @@ -36,18 +39,12 @@ var ( hubIdx map[string]map[string]Item ) +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted. type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` -} - -type ItemHubStatus struct { - Name string `json:"name"` - LocalVersion string `json:"local_version"` - LocalPath string `json:"local_path"` - Description string `json:"description"` - UTF8Status string `json:"utf8_status"` - Status string `json:"status"` + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? } // Item can be: parser, scenario, collection.. @@ -84,7 +81,7 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } -func (i *Item) status() (string, emoji.Emoji) { +func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false @@ -124,19 +121,6 @@ func (i *Item) status() (string, emoji.Emoji) { return status, emo } -func (i *Item) hubStatus() ItemHubStatus { - status, emo := i.status() - - return ItemHubStatus{ - Name: i.Name, - LocalVersion: i.LocalVersion, - LocalPath: i.LocalPath, - Description: i.Description, - Status: status, - UTF8Status: fmt.Sprintf("%v %s", emo, status), - } -} - // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) @@ -206,6 +190,23 @@ func GetItem(itemType string, itemName string) *Item { return nil } +// GetItemNames returns the list of item (full) names for a given type +// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// The names can be used to retrieve the item with GetItem() +func GetItemNames(itemType string) []string { + m := GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + func AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { @@ -257,32 +258,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) { return retStr, nil } - -// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy) -func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { - if _, ok := hubIdx[itemType]; !ok { - log.Errorf("type %s doesn't exist", itemType) - - return nil - } - - ret := make([]ItemHubStatus, 0) - - // remember, you do it for the user :) - for _, item := range hubIdx[itemType] { - if name != "" && name != item.Name { - // user has requested a specific name - continue - } - // Only enabled items ? - if !all && !item.Installed { - continue - } - // Check the item status - ret = append(ret, item.hubStatus()) - } - - sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) - - return ret -} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 2fef828c2b2..9fafb3f1c7f 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) { item.Local = false item.Tainted = false - txt, _ := item.status() + txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) item.Installed = false @@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) { item.Local = true item.Tainted = false - txt, _ = item.status() + txt, _ = item.Status() require.Equal(t, "disabled,local", txt) } @@ -273,10 +273,8 @@ func TestInstallParser(t *testing.T) { for _, it := range hubIdx[PARSERS] { testInstallItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) testTaintItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) testUpdateItem(cfg.Hub, t, it) it = hubIdx[PARSERS][it.Name] testDisableItem(cfg.Hub, t, it) @@ -309,11 +307,6 @@ func TestInstallCollection(t *testing.T) { testUpdateItem(cfg.Hub, t, it) it = hubIdx[COLLECTIONS][it.Name] testDisableItem(cfg.Hub, t, it) - - it = hubIdx[COLLECTIONS][it.Name] - x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) - log.Infof("%+v", x) - break } } diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 1840d14ec2f..77c3dc089a9 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -79,41 +79,42 @@ teardown() { assert_output "$expected" } - @test "cscli parsers list [parser]..." { + # non-existent + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" + + # not installed + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp 'crowdsecurity/whitelists.*disabled' + + # install two items rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth - # list one item + # list an installed item rune -0 cscli parsers list crowdsecurity/whitelists - assert_output --partial "crowdsecurity/whitelists" + assert_output --regexp "crowdsecurity/whitelists.*enabled" refute_output --partial "crowdsecurity/windows-auth" - # list multiple items - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth + # list multiple installed and non installed items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs assert_output --partial "crowdsecurity/whitelists" assert_output --partial "crowdsecurity/windows-auth" + assert_output --partial "crowdsecurity/traefik-logs" rune -0 cscli parsers list crowdsecurity/whitelists -o json rune -0 jq '.parsers | length' <(output) assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o json + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json rune -0 jq '.parsers | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli parsers list crowdsecurity/whitelists -o raw rune -0 grep -vc 'name,status,version,description' <(output) assert_output "1" - rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o raw + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli parsers list [parser]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli parsers list crowdsecurity/whitelists - # not existing - rune -1 cscli parsers list blahblah/blahblah + assert_output "3" } @test "cscli parsers install [parser]..." { diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index ecec032d178..2eeb6146bc2 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -77,41 +77,42 @@ teardown() { assert_output "$expected" } - @test "cscli scenarios list [scenario]..." { + # non-existent + rune -1 cscli scenario install foo/bar + assert_stderr --partial "can't find 'foo/bar' in scenarios" + + # not installed + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp 'crowdsecurity/ssh-bf.*disabled' + + # install two items rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf - # list one item + # list an installed item rune -0 cscli scenarios list crowdsecurity/ssh-bf - assert_output --partial "crowdsecurity/ssh-bf" + assert_output --regexp "crowdsecurity/ssh-bf.*enabled" refute_output --partial "crowdsecurity/telnet-bf" - # list multiple items - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf + # list multiple installed and non installed items + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf assert_output --partial "crowdsecurity/ssh-bf" assert_output --partial "crowdsecurity/telnet-bf" + assert_output --partial "crowdsecurity/aws-bf" rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json rune -0 jq '.scenarios | length' <(output) assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json rune -0 jq '.scenarios | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw rune -0 grep -vc 'name,status,version,description' <(output) assert_output "1" - rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli scenarios list [scenario]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli scenarios list crowdsecurity/ssh-bf - # not existing - rune -1 cscli scenarios list blahblah/blahblah + assert_output "3" } @test "cscli scenarios install [scenario]..." { From 810a8adcf0057fa304fe2eae09ed45a0bdafff1a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:35:31 +0200 Subject: [PATCH 096/263] fix build (#2548) --- cmd/crowdsec-cli/collections.go | 5 ----- cmd/crowdsec-cli/item_metrics.go | 8 ++++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 8dbb41fcaf7..158b808234a 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -254,11 +254,6 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error { return err } - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - for _, name := range args { if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { return err diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index 1eb8e39e871..ec925262e9d 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -21,18 +21,18 @@ import ( func ShowMetrics(hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: - metrics := GetParserMetric(prometheusURL, hubItem.Name) + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) parserMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.SCENARIOS: - metrics := GetScenarioMetric(prometheusURL, hubItem.Name) + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) scenarioMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.COLLECTIONS: for _, item := range hubItem.Parsers { - metrics := GetParserMetric(prometheusURL, item) + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, item) parserMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(prometheusURL, item) + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, item) scenarioMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Collections { From 4eae40865e8436fc24a69d9448f2202a9a5ba3e4 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:17:37 +0200 Subject: [PATCH 097/263] HubIndex struct, comments, name changes (#2549) * pkg/cwhub: rename PARSERS_OVFLW -> POSTOVERFLOWS * mostly comments, some light cleanup * move type hubtest.HubIndex -> cwhub.HubIndex * move and rename LoadPkgIndex -> ParseIndex * move displaySummary(), skippedLocal, skippedTainted to HubIndex struct --- cmd/crowdsec-cli/config_backup.go | 2 +- cmd/crowdsec-cli/config_restore.go | 2 +- cmd/crowdsec-cli/hub.go | 4 +- cmd/crowdsec-cli/postoverflows.go | 30 ++++----- cmd/crowdsec-cli/support.go | 2 +- pkg/cwhub/cwhub.go | 52 +++++---------- pkg/cwhub/cwhub_test.go | 48 ++++++------- pkg/cwhub/download.go | 14 ++-- pkg/cwhub/helpers.go | 15 +++-- pkg/cwhub/helpers_test.go | 92 ++++++++++++------------- pkg/cwhub/hubindex.go | 104 +++++++++++++++++++++++++++++ pkg/cwhub/install.go | 10 +-- pkg/cwhub/loader.go | 95 ++++++-------------------- pkg/hubtest/coverage.go | 8 +-- pkg/hubtest/hubtest.go | 6 +- pkg/hubtest/hubtest_item.go | 14 ++-- pkg/parser/unix_parser.go | 4 +- pkg/setup/install.go | 2 +- 18 files changed, 268 insertions(+), 236 deletions(-) create mode 100644 pkg/cwhub/hubindex.go diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 436eff8aa97..73d1aca03db 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -44,7 +44,7 @@ func backupHub(dirPath string) error { //for the local/tainted ones, we backup the full file if v.Tainted || v.Local || !v.UpToDate { //we need to backup stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { + if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index a528372effb..377f48a2d85 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -92,7 +92,7 @@ func restoreHub(dirPath string) error { if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { continue } - if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { + if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS { //we expect a stage here if !file.IsDir() { continue diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index bd0f6f77b2b..c6afbcd1425 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -63,7 +63,7 @@ func runHubList(cmd *cobra.Command, args []string) error { cwhub.DisplaySummary() err = ListItems(color.Output, []string{ - cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, + cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, }, nil, true, false, all) if err != nil { return err @@ -164,7 +164,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { } log.Infof("Upgrading postoverflows") - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil { + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { return err } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index 7ff245a9f92..8c5f1b1754e 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -67,15 +67,15 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { } for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name) + t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name) if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name) - Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError) + nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) + Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, force, downloadOnly); err != nil { + if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -95,7 +95,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete) + return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete) }, RunE: runPostOverflowsInstall, } @@ -127,7 +127,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { } if all { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, force) + err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force) if err != nil { return err } @@ -140,7 +140,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, force) + err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force) if err != nil { return err } @@ -158,7 +158,7 @@ func NewPostOverflowsRemoveCmd() *cobra.Command { Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) }, RunE: runPostOverflowsRemove, } @@ -185,7 +185,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil { + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { return err } return nil @@ -196,7 +196,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, force); err != nil { + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil { return err } } @@ -212,7 +212,7 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command { Example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) }, RunE: runPostOverflowUpgrade, } @@ -242,7 +242,7 @@ func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err = InspectItem(name, cwhub.PARSERS_OVFLW, noMetrics); err != nil { + if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil { return err } } @@ -259,13 +259,13 @@ func NewPostOverflowsInspectCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) }, RunE: runPostOverflowsInspect, } flags := cmdPostOverflowsInspect.Flags() - // XXX: is this needed for postoverflows? + flags.StringP("url", "u", "", "Prometheus url") flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") @@ -280,7 +280,7 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error { return err } - if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil { + if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil { return err } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 6477383721d..0c87ad3c97d 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -333,7 +333,7 @@ cscli support dump -f /tmp/crowdsec-support.zip if !skipHub { infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) - infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW) + infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS) infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index bf4d69d6dcc..580bf4e8196 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -12,31 +12,15 @@ import ( "github.com/enescakir/emoji" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" ) -const ( - HubIndexFile = ".index.json" - - // managed item types - PARSERS = "parsers" - PARSERS_OVFLW = "postoverflows" - SCENARIOS = "scenarios" - COLLECTIONS = "collections" -) var ( - ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS} - ErrMissingReference = errors.New("Reference(s) missing in collection") - // XXX: can we remove these globals? - skippedLocal = 0 - skippedTainted = 0 RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" HubBranch = "master" - hubIdx map[string]map[string]Item ) // ItemVersion is used to detect the version of a given item @@ -47,16 +31,16 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? } -// Item can be: parser, scenario, collection.. +// Item represents an object managed in the hub. It can be a parser, scenario, collection.. type Item struct { // descriptive info Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name" + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json - Author string `json:"author,omitempty"` // as seen in .config.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any // remote (hub) info @@ -74,13 +58,15 @@ type Item struct { Tainted bool `json:"tainted,omitempty"` // has it been locally modified Local bool `json:"local,omitempty"` // if it's a non versioned control one - // if it's a collection, it's not a single file + // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } +// Status returns the status of the item as a string and an emoji +// ie. "enabled,update-available" and emoji.Warning func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false @@ -126,8 +112,9 @@ func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) } +// GetItemMap returns the map of items for a given type func GetItemMap(itemType string) map[string]Item { - m, ok := hubIdx[itemType] + m, ok := hubIdx.Items[itemType] if !ok { return nil } @@ -135,7 +122,7 @@ func GetItemMap(itemType string) map[string]Item { return m } -// Given a FileInfo, extract the map key. Follow a symlink if necessary +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary func itemKey(itemPath string) (string, error) { f, err := os.Lstat(itemPath) if err != nil { @@ -182,6 +169,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { return &v, nil } +// GetItem returns the item from hub based on its type and full name (author/name) func GetItem(itemType string, itemName string) *Item { if m, ok := GetItemMap(itemType)[itemName]; ok { return &m @@ -207,10 +195,11 @@ func GetItemNames(itemType string) []string { return names } +// AddItem adds an item to the hub index func AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { - hubIdx[itemType][item.Name] = item + hubIdx.Items[itemType][item.Name] = item return nil } } @@ -218,17 +207,9 @@ func AddItem(itemType string, item Item) error { return fmt.Errorf("ItemType %s is unknown", itemType) } -func DisplaySummary() { - log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW])) - - if skippedLocal > 0 || skippedTainted > 0 { - log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted) - } -} - +// GetInstalledItems returns the list of installed items func GetInstalledItems(itemType string) ([]Item, error) { - items, ok := hubIdx[itemType] + items, ok := hubIdx.Items[itemType] if !ok { return nil, fmt.Errorf("no %s in hubIdx", itemType) } @@ -244,6 +225,7 @@ func GetInstalledItems(itemType string) ([]Item, error) { return retItems, nil } +// GetInstalledItemsAsString returns the names of the installed items func GetInstalledItemsAsString(itemType string) ([]string, error) { items, err := GetInstalledItems(itemType) if err != nil { diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 9fafb3f1c7f..da85ed65390 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -180,9 +180,9 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) err = EnableItem(cfg, &item) require.NoError(t, err, "failed to enable %s", item.Name) @@ -190,11 +190,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) @@ -208,11 +208,11 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status err := DownloadLatest(cfg, &item, true, true) @@ -222,12 +222,12 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove err := DisableItem(cfg, &item, false, false) @@ -238,9 +238,9 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge err = DisableItem(cfg, &item, true, false) @@ -251,8 +251,8 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { @@ -270,15 +270,15 @@ func TestInstallParser(t *testing.T) { getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx[PARSERS] { + for _, it := range hubIdx.Items[PARSERS] { testInstallItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] testTaintItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] testUpdateItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] testDisableItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] break } @@ -299,13 +299,13 @@ func TestInstallCollection(t *testing.T) { getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx[COLLECTIONS] { + for _, it := range hubIdx.Items[COLLECTIONS] { testInstallItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testTaintItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testUpdateItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testDisableItem(cfg.Hub, t, it) break } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index ef111ba62b9..70f7ab35213 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -19,20 +19,21 @@ import ( var ErrIndexNotFound = fmt.Errorf("index not found") +// UpdateHubIdx downloads the latest version of the index and updates the one in memory func UpdateHubIdx(hub *csconfig.Hub) error { bidx, err := DownloadHubIdx(hub) if err != nil { return fmt.Errorf("failed to download index: %w", err) } - ret, err := LoadPkgIndex(bidx) + ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { return fmt.Errorf("failed to read index: %w", err) } } - hubIdx = ret + hubIdx = HubIndex{Items: ret} if _, err := LocalSync(hub); err != nil { return fmt.Errorf("failed to sync: %w", err) @@ -41,6 +42,7 @@ func UpdateHubIdx(hub *csconfig.Hub) error { return nil } +// DownloadHubIdx downloads the latest version of the index and returns the content func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) @@ -115,7 +117,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] + val, ok := hubIdx.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -151,7 +153,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly } } - hubIdx[ptrtype][p] = val + hubIdx.Items[ptrtype][p] = val } } @@ -265,11 +267,12 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } +// DownloadDataIfNeeded downloads the data files for an item func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) @@ -287,6 +290,7 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { return nil } +// downloadData downloads the data files for an item func downloadData(dataFolder string, force bool, reader io.Reader) error { var err error diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index ad9032f8140..2db76863638 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -12,12 +12,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) -// pick a hub branch corresponding to the current crowdsec version. +// chooseHubBranch returns the branch name to use for the hub +// It can be "master" or branch corresponding to the current crowdsec version func chooseHubBranch() string { latest, err := cwversion.Latest() if err != nil { log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - //lint:ignore nilerr return "master" } @@ -61,8 +61,9 @@ func SetHubBranch() { log.Debugf("Using branch '%s' for the hub", HubBranch) } -func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error { - item := GetItem(obtype, name) +// InstallItem installs an item from the hub +func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error { + item := GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) } @@ -80,7 +81,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = AddItem(obtype, *item); err != nil { + if err = AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -94,7 +95,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := AddItem(obtype, *item); err != nil { + if err := AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -103,6 +104,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return nil } +// RemoveItem removes one - or all - the items from the hub func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error { if name != "" { item := GetItem(itemType, name) @@ -151,6 +153,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo return nil } +// UpgradeConfig upgrades an item from the hub func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error { updated := 0 found := false diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index e7e1314c6c2..ecb778fdc54 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -15,19 +15,19 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -40,17 +40,17 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { getHubIdxOrFail(t) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } // Install a collection, disable a scenario. @@ -62,17 +62,17 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) @@ -80,11 +80,11 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) if err = UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) @@ -94,7 +94,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.NoError(t, err) getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) } func getHubIdxOrFail(t *testing.T) { @@ -113,17 +113,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) @@ -131,12 +131,12 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario @@ -147,21 +147,21 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * t.Fatalf("failed to download index : %s", err) } - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) getHubIdxOrFail(t) err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } func assertCollectionDepsInstalled(t *testing.T, collection string) { t.Helper() - c := hubIdx[COLLECTIONS][collection] + c := hubIdx.Items[COLLECTIONS][collection] require.NoError(t, CollecDepsCheck(&c)) } diff --git a/pkg/cwhub/hubindex.go b/pkg/cwhub/hubindex.go new file mode 100644 index 00000000000..74612257df3 --- /dev/null +++ b/pkg/cwhub/hubindex.go @@ -0,0 +1,104 @@ +package cwhub + +import ( + "encoding/json" + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + + +const ( + HubIndexFile = ".index.json" + + // managed item types + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" +) + +var ( + // XXX: The order is important, as it is used to construct the + // index tree in memory --> collections must be last + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} + hubIdx = HubIndex{} +) + + +type HubItems map[string]map[string]Item + +type HubIndex struct { + Items HubItems + skippedLocal int + skippedTainted int +} + +// displaySummary prints a total count of the hub items +func (h HubIndex) displaySummary() { + msg := "Loaded: " + for itemType := range h.Items { + msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + } + log.Info(strings.Trim(msg, ", ")) + + if h.skippedLocal > 0 || h.skippedTainted > 0 { + log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted) + } +} + +// DisplaySummary prints a total count of the hub items. +// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton +func DisplaySummary() { + hubIdx.displaySummary() +} + +// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections +func ParseIndex(buff []byte) (HubItems, error) { + var ( + RawIndex HubItems + missingItems []string + ) + + if err := json.Unmarshal(buff, &RawIndex); err != nil { + return nil, fmt.Errorf("failed to unmarshal index: %w", err) + } + + log.Debugf("%d item types in hub index", len(ItemTypes)) + + // Iterate over the different types to complete the struct + for _, itemType := range ItemTypes { + log.Tracef("%s: %d items", itemType, len(RawIndex[itemType])) + + for name, item := range RawIndex[itemType] { + item.Name = name + item.Type = itemType + x := strings.Split(item.RemotePath, "/") + item.FileName = x[len(x)-1] + RawIndex[itemType][name] = item + + if itemType != COLLECTIONS { + continue + } + + // if it's a collection, check its sub-items are present + // XXX should be done later + for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if _, ok := RawIndex[ptrtype][p]; !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) + missingItems = append(missingItems, p) + } + } + } + } + } + + if len(missingItems) > 0 { + return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) + } + + return RawIndex, nil +} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 45e2ba41977..a338d868b99 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -20,7 +20,7 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { target.Downloaded = false log.Infof("Removed source file [%s]: %s", target.Name, itempath) - hubIdx[target.Type][target.Name] = target + hubIdx.Items[target.Type][target.Name] = target return target, nil } @@ -54,7 +54,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - if val, ok := hubIdx[ptrtype][p]; ok { + if val, ok := hubIdx.Items[ptrtype][p]; ok { // check if the item doesn't belong to another collection before removing it toRemove := true @@ -130,7 +130,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error } } - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } @@ -172,7 +172,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error { for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] + val, ok := hubIdx.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -208,7 +208,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error { log.Infof("Enabled %s : %s", target.Type, target.Name) target.Installed = true - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 8b470148c1e..a989ee3eedf 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,7 +2,6 @@ package cwhub import ( "crypto/sha256" - "encoding/json" "errors" "fmt" "io" @@ -131,8 +130,8 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { } else if ret.stage == COLLECTIONS { ret.ftype = COLLECTIONS ret.stage = "" - } else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW { - // its a PARSER / PARSER_OVFLW with a stage + } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { + // its a PARSER / POSTOVERFLOW with a stage return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) } @@ -141,7 +140,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { return ret, inhub, nil } -func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { +func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { var ( local bool hubpath string @@ -198,12 +197,12 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { // if it's not a symlink and not in hub, it's a local file, don't bother if local && !inhub { log.Tracef("%s is a local file, skip", path) - skippedLocal++ + hubIdx.skippedLocal++ // log.Infof("local scenario, skip.") _, fileName := filepath.Split(path) - hubIdx[info.ftype][info.fname] = Item{ + hubIdx.Items[info.ftype][info.fname] = Item{ Name: info.fname, Stage: info.stage, Installed: true, @@ -222,7 +221,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { match := false - for name, item := range hubIdx[info.ftype] { + for name, item := range hubIdx.Items[info.ftype] { log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") if info.fname != item.FileName { @@ -304,7 +303,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { if !match { log.Tracef("got tainted match for %s: %s", item.Name, path) - skippedTainted++ + hubIdx.skippedTainted++ // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { item.LocalPath = path @@ -317,14 +316,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { item.LocalHash = sha } - // update the entry if appropriate - // if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D { - // fmt.Printf("Updating %s", k) - // hubIdx[ftype][k] = v - // } else if !inhub { - - // } else if - hubIdx[info.ftype][name] = item + hubIdx.Items[info.ftype][name] = item return nil } @@ -350,7 +342,7 @@ func CollecDepsCheck(v *Item) error { for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { sliceType := ItemTypes[idx] for _, subName := range itemSlice { - subItem, ok := hubIdx[sliceType][subName] + subItem, ok := hubIdx.Items[sliceType][subName] if !ok { return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) } @@ -372,7 +364,7 @@ func CollecDepsCheck(v *Item) error { return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - hubIdx[sliceType][subName] = subItem + hubIdx.Items[sliceType][subName] = subItem } // propagate the state of sub-items to set @@ -403,7 +395,7 @@ func CollecDepsCheck(v *Item) error { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } - hubIdx[sliceType][subName] = subItem + hubIdx.Items[sliceType][subName] = subItem log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) } @@ -415,20 +407,20 @@ func CollecDepsCheck(v *Item) error { func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { warnings := []string{} - // For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last + // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s : %s", cpath, err) } - err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit) + err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit) if err != nil { return warnings, err } } - for name, item := range hubIdx[COLLECTIONS] { + for name, item := range hubIdx.Items[COLLECTIONS] { if !item.Installed { continue } @@ -438,7 +430,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { case 0: // latest if err := CollecDepsCheck(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - hubIdx[COLLECTIONS][name] = item + hubIdx.Items[COLLECTIONS][name] = item } case 1: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) @@ -454,8 +446,8 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { // Updates the info from HubInit() with the local state func LocalSync(hub *csconfig.Hub) ([]string, error) { - skippedLocal = 0 - skippedTainted = 0 + hubIdx.skippedLocal = 0 + hubIdx.skippedTainted = 0 warnings, err := SyncDir(hub, hub.InstallDir) if err != nil { @@ -482,7 +474,7 @@ func GetHubIdx(hub *csconfig.Hub) error { return fmt.Errorf("unable to read index file: %w", err) } - ret, err := LoadPkgIndex(bidx) + ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { return fmt.Errorf("unable to load existing index: %w", err) @@ -492,7 +484,7 @@ func GetHubIdx(hub *csconfig.Hub) error { return err } - hubIdx = ret + hubIdx = HubIndex{Items: ret} _, err = LocalSync(hub) if err != nil { @@ -501,52 +493,3 @@ func GetHubIdx(hub *csconfig.Hub) error { return nil } - -// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections -func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { - var ( - RawIndex map[string]map[string]Item - missingItems []string - ) - - if err := json.Unmarshal(buff, &RawIndex); err != nil { - return nil, fmt.Errorf("failed to unmarshal index: %w", err) - } - - log.Debugf("%d item types in hub index", len(ItemTypes)) - - // Iterate over the different types to complete the struct - for _, itemType := range ItemTypes { - log.Tracef("%d item", len(RawIndex[itemType])) - - for name, item := range RawIndex[itemType] { - item.Name = name - item.Type = itemType - x := strings.Split(item.RemotePath, "/") - item.FileName = x[len(x)-1] - RawIndex[itemType][name] = item - - if itemType != COLLECTIONS { - continue - } - - // if it's a collection, check its sub-items are present - // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } - } - } - } - } - - if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) - } - - return RawIndex, nil -} diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index eeff24b57b6..29db527150e 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -27,12 +27,12 @@ type ScenarioCoverage struct { func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { var coverage []ParserCoverage - if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok { + if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { return coverage, fmt.Errorf("no parsers in hub index") } //populate from hub, iterate in alphabetical order var pkeys []string - for pname := range h.HubIndex.Data[cwhub.PARSERS] { + for pname := range h.HubIndex.Items[cwhub.PARSERS] { pkeys = append(pkeys, pname) } sort.Strings(pkeys) @@ -100,12 +100,12 @@ func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { var coverage []ScenarioCoverage - if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok { + if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { return coverage, fmt.Errorf("no scenarios in hub index") } //populate from hub, iterate in alphabetical order var pkeys []string - for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] { + for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] { pkeys = append(pkeys, scenarioName) } sort.Strings(pkeys) diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index c1aa4251ca1..eff2aa8fb0e 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -18,7 +18,7 @@ type HubTest struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.HubIndex Tests []*HubTestItem } @@ -62,7 +62,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } // load hub index - hubIndex, err := cwhub.LoadPkgIndex(bidx) + hubIndex, err := cwhub.ParseIndex(bidx) if err != nil { return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err) } @@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &HubIndex{Data: hubIndex}, + HubIndex: &cwhub.HubIndex{Items: hubIndex}, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 47a15122059..1abf466501b 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -25,10 +25,6 @@ type HubTestItemConfig struct { OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } -type HubIndex struct { - Data map[string]map[string]cwhub.Item -} - type HubTestItem struct { Name string Path string @@ -56,7 +52,7 @@ type HubTestItem struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.HubIndex Config *HubTestItemConfig @@ -148,7 +144,7 @@ func (t *HubTestItem) InstallHub() error { continue } var parserDirDest string - if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok { + if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) @@ -232,7 +228,7 @@ func (t *HubTestItem) InstallHub() error { continue } var scenarioDirDest string - if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok { + if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) @@ -301,7 +297,7 @@ func (t *HubTestItem) InstallHub() error { continue } var postoverflowDirDest string - if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok { + if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) @@ -423,7 +419,7 @@ func (t *HubTestItem) InstallHub() error { } // install data for postoverflows if needed - ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW) + ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { if item.Installed { if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 2e4a8035ba0..48b09795bdf 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -64,7 +64,7 @@ func NewParsers() *Parsers { StageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0), } - for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { + for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { for _, hubParserItem := range cwhub.GetItemMap(itemType) { if hubParserItem.Installed { stagefile := Stagefile{ @@ -74,7 +74,7 @@ func NewParsers() *Parsers { if itemType == cwhub.PARSERS { parsers.StageFiles = append(parsers.StageFiles, stagefile) } - if itemType == cwhub.PARSERS_OVFLW { + if itemType == cwhub.POSTOVERFLOWS { parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) } } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 92a1968c85e..53ab46b2a3c 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -121,7 +121,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { + if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } } From be6555e46c36a36147026e9d3d2586aaeaed59c1 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 18 Oct 2023 09:38:33 +0200 Subject: [PATCH 098/263] Refact pkg/csconfig, HubCfg (#2552) - rename csconfig.Hub -> HubCfg - move some Load*() functions to NewConfig() - config.yaml: optional common section - remove unused working_dir --- cmd/crowdsec-cli/config_restore.go | 4 -- cmd/crowdsec-cli/hub.go | 4 -- cmd/crowdsec-cli/main.go | 4 +- cmd/crowdsec-cli/require/require.go | 4 -- cmd/crowdsec/main.go | 11 +--- config/config.yaml | 1 - config/config_win.yaml | 1 - config/config_win_no_lapi.yaml | 1 - config/dev.yaml | 1 - config/user.yaml | 1 - docker/config.yaml | 1 - pkg/csconfig/api.go | 4 -- pkg/csconfig/api_test.go | 8 +-- pkg/csconfig/common.go | 11 ++-- pkg/csconfig/common_test.go | 83 --------------------------- pkg/csconfig/config.go | 24 ++++++-- pkg/csconfig/config_paths.go | 6 +- pkg/csconfig/crowdsec_service.go | 4 -- pkg/csconfig/crowdsec_service_test.go | 41 ++++++------- pkg/csconfig/cscli.go | 5 +- pkg/csconfig/cscli_test.go | 30 ++-------- pkg/csconfig/hub.go | 20 +++---- pkg/csconfig/hub_test.go | 44 +++----------- pkg/csconfig/simulation.go | 5 -- pkg/csconfig/simulation_test.go | 13 +---- pkg/csconfig/testdata/config.yaml | 1 - pkg/cwhub/cwhub_test.go | 10 ++-- pkg/cwhub/download.go | 10 ++-- pkg/cwhub/download_test.go | 6 +- pkg/cwhub/hubindex.go | 2 + pkg/cwhub/install.go | 6 +- pkg/cwhub/loader.go | 8 +-- pkg/hubtest/hubtest_item.go | 4 +- pkg/setup/install.go | 4 -- test/bats/01_crowdsec.bats | 10 ++-- 35 files changed, 105 insertions(+), 287 deletions(-) delete mode 100644 pkg/csconfig/common_test.go diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 377f48a2d85..ccd9ebc5ed4 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -48,10 +48,6 @@ func silentInstallItem(name string, obtype string) (string, error) { func restoreHub(dirPath string) error { var err error - if err := csConfig.LoadHub(); err != nil { - return err - } - cwhub.SetHubBranch() for _, itype := range cwhub.ItemTypes { diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c6afbcd1425..501fbc498b9 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -88,10 +88,6 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadHub(); err != nil { - return err - } - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index e80e8cb4142..1ddac4bf80c 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -51,10 +51,8 @@ func initConfig() { if err != nil { log.Fatal(err) } - if err := csConfig.LoadCSCLI(); err != nil { - log.Fatal(err) - } } else { + // XXX: check all the defaults csConfig = csconfig.NewDefaultConfig() } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index f4129a44f8f..fba59cb1e40 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -65,10 +65,6 @@ func Notifications(c *csconfig.Config) error { } func Hub (c *csconfig.Config) error { - if err := c.LoadHub(); err != nil { - return err - } - if c.Hub == nil { return fmt.Errorf("you must configure cli before interacting with hub") } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c604e670a84..e93dbef044c 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) { cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet) if err != nil { - return nil, err - } - - if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) { - return nil, fmt.Errorf("unable to load configuration: common section is empty") + return nil, fmt.Errorf("while loading configuration file: %w", err) } cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags) @@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo dumpStates = true } - // Configuration paths are dependency to load crowdsec configuration - if err := cConfig.LoadConfigurationPaths(); err != nil { - return nil, err - } - if flags.SingleFileType != "" && flags.OneShotDSN != "" { // if we're in time-machine mode, we don't want to log to file cConfig.Common.LogMedia = "stdout" diff --git a/config/config.yaml b/config/config.yaml index 232b0bc4389..2b0e4dfca1a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -6,7 +6,6 @@ common: log_max_size: 20 compress_logs: true log_max_files: 10 - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/config/config_win.yaml b/config/config_win.yaml index 7863f4fdd50..5c34c69a2c0 100644 --- a/config/config_win.yaml +++ b/config/config_win.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/config_win_no_lapi.yaml b/config/config_win_no_lapi.yaml index 35c7f2c6f11..af240228bb5 100644 --- a/config/config_win_no_lapi.yaml +++ b/config/config_win_no_lapi.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/dev.yaml b/config/dev.yaml index 2ff62506017..2123dc858d4 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -2,7 +2,6 @@ common: daemonize: true log_media: stdout log_level: info - working_dir: . config_paths: config_dir: ./config data_dir: ./data/ diff --git a/config/user.yaml b/config/user.yaml index 67bdfa3fc49..a1047dcd0f7 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data diff --git a/docker/config.yaml b/docker/config.yaml index 5259a0fe26e..6811329099a 100644 --- a/docker/config.yaml +++ b/docker/config.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index bbe2e16225f..c1577782f9c 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error { log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs)) } - if err := c.LoadCommon(); err != nil { - return fmt.Errorf("loading common configuration: %s", err) - } - c.API.Server.LogDir = c.Common.LogDir c.API.Server.LogMedia = c.Common.LogMedia c.API.Server.CompressLogs = c.Common.CompressLogs diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index 4338de9c1ee..10128b76bc8 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -3,7 +3,6 @@ package csconfig import ( "net" "os" - "path/filepath" "strings" "testing" @@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) { err := tmpLAPI.LoadProfiles() require.NoError(t, err) - LogDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - logLevel := log.InfoLevel config := &Config{} fcontent, err := os.ReadFile("./testdata/config.yaml") @@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) { DbPath: "./testdata/test.db", }, Common: &CommonCfg{ - LogDir: "./testdata/", + LogDir: "./testdata", LogMedia: "stdout", }, DisableAPI: false, @@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) { ShareContext: ptr.Of(false), ConsoleManagement: ptr.Of(false), }, - LogDir: LogDirFullPath, + LogDir: "./testdata", LogMedia: "stdout", OnlineClient: &OnlineApiClientCfg{ CredentialsFilePath: "./testdata/online-api-secrets.yaml", diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 9d80cd95ac1..7e1ef6e5c98 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -14,7 +14,7 @@ type CommonCfg struct { LogMedia string `yaml:"log_media"` LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file LogLevel *log.Level `yaml:"log_level"` - WorkingDir string `yaml:"working_dir,omitempty"` ///var/run + WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later CompressLogs *bool `yaml:"compress_logs,omitempty"` LogMaxSize int `yaml:"log_max_size,omitempty"` LogMaxAge int `yaml:"log_max_age,omitempty"` @@ -22,15 +22,18 @@ type CommonCfg struct { ForceColorLogs bool `yaml:"force_color_logs,omitempty"` } -func (c *Config) LoadCommon() error { +func (c *Config) loadCommon() error { var err error if c.Common == nil { - return fmt.Errorf("no common block provided in configuration file") + c.Common = &CommonCfg{} + } + + if c.Common.LogMedia == "" { + c.Common.LogMedia = "stdout" } var CommonCleanup = []*string{ &c.Common.LogDir, - &c.Common.WorkingDir, } for _, k := range CommonCleanup { if *k == "" { diff --git a/pkg/csconfig/common_test.go b/pkg/csconfig/common_test.go deleted file mode 100644 index 2c5f798a621..00000000000 --- a/pkg/csconfig/common_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package csconfig - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadCommon(t *testing.T) { - pidDirPath := "./testdata" - LogDirFullPath, err := filepath.Abs("./testdata/log/") - require.NoError(t, err) - - WorkingDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - tests := []struct { - name string - input *Config - expected *CommonCfg - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - WorkingDir: "./testdata/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - WorkingDir: WorkingDirFullPath, - }, - }, - { - name: "empty working dir", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - }, - }, - { - name: "no common", - input: &Config{}, - expected: nil, - expectedErr: "no common block provided in configuration file", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCommon() - cstest.RequireErrorContains(t, err, tc.expectedErr) - if tc.expectedErr != "" { - return - } - - assert.Equal(t, tc.expected, tc.input.Common) - }) - } -} diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 25f6f716ddd..a13ce245c51 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -36,7 +36,7 @@ type Config struct { PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"` DisableAPI bool `yaml:"-"` DisableAgent bool `yaml:"-"` - Hub *Hub `yaml:"-"` + Hub *HubCfg `yaml:"-"` } func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { @@ -73,18 +73,34 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort) } + if err = cfg.loadCommon(); err != nil { + return nil, "", err + } + + if err = cfg.loadConfigurationPaths(); err != nil { + return nil, "", err + } + + if err = cfg.loadHub(); err != nil { + return nil, "", err + } + + if err = cfg.loadCSCLI(); err != nil { + return nil, "", err + } + return &cfg, configData, nil } +// XXX: We must not not have a different behavior with an empty vs a missing configuration file. +// XXX: For this reason, all defaults have to come from NewConfig(). The following function should +// XXX: be replaced func NewDefaultConfig() *Config { logLevel := log.InfoLevel commonCfg := CommonCfg{ Daemonize: false, - PidDir: "/tmp/", LogMedia: "stdout", - //LogDir unneeded LogLevel: &logLevel, - WorkingDir: ".", } prometheus := PrometheusCfg{ Enabled: true, diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 24ff454b78d..07db4bd710c 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -15,21 +15,25 @@ type ConfigurationPaths struct { NotificationDir string `yaml:"notification_dir,omitempty"` } -func (c *Config) LoadConfigurationPaths() error { +func (c *Config) loadConfigurationPaths() error { var err error if c.ConfigPaths == nil { + // XXX: test me return fmt.Errorf("no configuration paths provided") } if c.ConfigPaths.DataDir == "" { + // XXX: test me return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section") } if c.ConfigPaths.HubDir == "" { + // XXX: test me c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub") } if c.ConfigPaths.HubIndexFile == "" { + // XXX: test me c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json") } diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 28d6e77f0ce..f9602a5be92 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -145,10 +145,6 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("loading api client: %s", err) } - if err := c.LoadHub(); err != nil { - return fmt.Errorf("while loading hub: %w", err) - } - c.Crowdsec.ContextToSend = make(map[string][]string, 0) fallback := false if c.Crowdsec.ConsoleContextPath == "" { diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index aa1d341f58c..06a7e91bd85 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) { acquisDirFullPath, err := filepath.Abs("./testdata/acquis") require.NoError(t, err) - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - contextFileFullPath, err := filepath.Abs("./testdata/context.yaml") require.NoError(t, err) @@ -66,10 +54,11 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: "", ConsoleContextPath: contextFileFullPath, AcquisitionFilePath: acquisFullPath, - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + ConfigDir: "./testdata", + DataDir: "./data", + HubDir: "./hub", + // XXX: need to ensure a default here + HubIndexFile: "", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -109,10 +98,11 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: acquisDirFullPath, AcquisitionFilePath: acquisFullPath, ConsoleContextPath: contextFileFullPath, - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, + ConfigDir: "./testdata", + // XXX: need to ensure a default here + HubIndexFile: "", + DataDir: "./data", + HubDir: "./hub", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -141,7 +131,7 @@ func TestLoadCrowdsec(t *testing.T) { }, }, Crowdsec: &CrowdsecServiceCfg{ - ConsoleContextPath: contextFileFullPath, + ConsoleContextPath: "./testdata/context.yaml", ConsoleContextValueLength: 10, }, }, @@ -149,10 +139,11 @@ func TestLoadCrowdsec(t *testing.T) { Enable: ptr.Of(true), AcquisitionDirPath: "", AcquisitionFilePath: "", - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, + ConfigDir: "./testdata", + // XXX: need to ensure a default here + HubIndexFile: "", + DataDir: "./data", + HubDir: "./hub", ConsoleContextPath: contextFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index a5ca39769ac..8db0a184872 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -19,13 +19,10 @@ type CscliCfg struct { PrometheusUrl string `yaml:"prometheus_uri"` } -func (c *Config) LoadCSCLI() error { +func (c *Config) loadCSCLI() error { if c.Cscli == nil { c.Cscli = &CscliCfg{} } - if err := c.LoadConfigurationPaths(); err != nil { - return err - } c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir c.Cscli.DataDir = c.ConfigPaths.DataDir c.Cscli.HubDir = c.ConfigPaths.HubDir diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index 03c3c95abe5..e928eda5a63 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -1,28 +1,14 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadCSCLI(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -46,25 +32,19 @@ func TestLoadCSCLI(t *testing.T) { }, }, expected: &CscliCfg{ - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + ConfigDir: "./testdata", + DataDir: "./data", + HubDir: "./hub", + HubIndexFile: "./hub/.index.json", PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, - { - name: "no configuration path", - input: &Config{}, - expected: &CscliCfg{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCSCLI() + err := tc.input.loadCSCLI() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index 4c3c610aad6..2164c19b25d 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,19 +1,15 @@ package csconfig -/*cscli specific config, such as hub directory*/ -type Hub struct { - HubIndexFile string - HubDir string - InstallDir string - InstallDataDir string +// HubConfig holds the configuration for a hub +type HubCfg struct { + HubIndexFile string // Path to the local index file + HubDir string // Where the hub items are downloaded + InstallDir string // Where to install items + InstallDataDir string // Where to install data } -func (c *Config) LoadHub() error { - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - - c.Hub = &Hub{ +func (c *Config) loadHub() error { + c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, InstallDir: c.ConfigPaths.ConfigDir, diff --git a/pkg/csconfig/hub_test.go b/pkg/csconfig/hub_test.go index d573e469044..0fa627ae0c1 100644 --- a/pkg/csconfig/hub_test.go +++ b/pkg/csconfig/hub_test.go @@ -1,32 +1,18 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadHub(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config - expected *Hub + expected *HubCfg expectedErr string }{ { @@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) { HubIndexFile: "./hub/.index.json", }, }, - expected: &Hub{ - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, - InstallDir: configDirFullPath, - InstallDataDir: dataFullPath, + expected: &HubCfg{ + HubDir: "./hub", + HubIndexFile: "./hub/.index.json", + InstallDir: "./testdata", + InstallDataDir: "./data", }, }, - { - name: "no data dir", - input: &Config{ - ConfigPaths: &ConfigurationPaths{ - ConfigDir: "./testdata", - HubDir: "./hub", - HubIndexFile: "./hub/.index.json", - }, - }, - expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section", - }, - { - name: "no configuration path", - input: &Config{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadHub() + err := tc.input.loadHub() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/simulation.go b/pkg/csconfig/simulation.go index 184708f0dc2..0d09aa478ff 100644 --- a/pkg/csconfig/simulation.go +++ b/pkg/csconfig/simulation.go @@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool { } func (c *Config) LoadSimulation() error { - - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - simCfg := SimulationConfig{} if c.ConfigPaths.SimulationFilePath == "" { c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml") diff --git a/pkg/csconfig/simulation_test.go b/pkg/csconfig/simulation_test.go index 44b8909a2b0..01f05e3975a 100644 --- a/pkg/csconfig/simulation_test.go +++ b/pkg/csconfig/simulation_test.go @@ -2,7 +2,6 @@ package csconfig import ( "fmt" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -12,12 +11,6 @@ import ( ) func TestSimulationLoading(t *testing.T) { - testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml") - require.NoError(t, err) - - badYamlFullPath, err := filepath.Abs("./testdata/config.yaml") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage), + expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage), }, { name: "basic bad file content", @@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, { name: "basic bad file content", @@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, } diff --git a/pkg/csconfig/testdata/config.yaml b/pkg/csconfig/testdata/config.yaml index 288c09b8429..17975b10501 100644 --- a/pkg/csconfig/testdata/config.yaml +++ b/pkg/csconfig/testdata/config.yaml @@ -2,7 +2,6 @@ common: daemonize: false log_media: stdout log_level: info - working_dir: . prometheus: enabled: true level: full diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index da85ed65390..7f3a50a1a50 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -121,7 +121,7 @@ func TestIndexDownload(t *testing.T) { } func getTestCfg() *csconfig.Config { - cfg := &csconfig.Config{Hub: &csconfig.Hub{}} + cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}} cfg.Hub.InstallDir, _ = filepath.Abs("./install") cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") @@ -172,7 +172,7 @@ func envTearDown(cfg *csconfig.Config) { } } -func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { +func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { // Install the parser err := DownloadLatest(cfg, &item, false, false) require.NoError(t, err, "failed to download %s", item.Name) @@ -193,7 +193,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } -func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { +func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) @@ -211,7 +211,7 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } -func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { +func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status @@ -226,7 +226,7 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } -func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { +func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 70f7ab35213..7b6771867dd 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -20,7 +20,7 @@ import ( var ErrIndexNotFound = fmt.Errorf("index not found") // UpdateHubIdx downloads the latest version of the index and updates the one in memory -func UpdateHubIdx(hub *csconfig.Hub) error { +func UpdateHubIdx(hub *csconfig.HubCfg) error { bidx, err := DownloadHubIdx(hub) if err != nil { return fmt.Errorf("failed to download index: %w", err) @@ -43,7 +43,7 @@ func UpdateHubIdx(hub *csconfig.Hub) error { } // DownloadHubIdx downloads the latest version of the index and returns the content -func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { +func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) @@ -98,7 +98,7 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { } // DownloadLatest will download the latest version of Item to the tdir directory -func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error { +func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -165,7 +165,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly return nil } -func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { +func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { tdir := hub.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files @@ -273,7 +273,7 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { } // DownloadDataIfNeeded downloads the data files for an item -func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { +func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 351b08f8e35..b1b41c5792f 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -17,7 +17,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "x" - ret, err := DownloadHubIdx(&csconfig.Hub{}) + ret, err := DownloadHubIdx(&csconfig.HubCfg{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { log.Errorf("unexpected error %s", err) } @@ -29,7 +29,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "https://baddomain/%s/%s" - ret, err = DownloadHubIdx(&csconfig.Hub{}) + ret, err = DownloadHubIdx(&csconfig.HubCfg{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { log.Errorf("unexpected error %s", err) } @@ -41,7 +41,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = back - ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"}) + ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { log.Errorf("unexpected error %s", err) } diff --git a/pkg/cwhub/hubindex.go b/pkg/cwhub/hubindex.go index 74612257df3..a29cd873a25 100644 --- a/pkg/cwhub/hubindex.go +++ b/pkg/cwhub/hubindex.go @@ -29,6 +29,8 @@ var ( type HubItems map[string]map[string]Item +// HubIndex represents the runtime status of the hub (parsed items, etc.) +// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared type HubIndex struct { Items HubItems skippedLocal int diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index a338d868b99..4ac955b7b47 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -10,7 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { +func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { itempath := hub.HubDir + "/" + target.RemotePath // disable hub file @@ -26,7 +26,7 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { } // DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error { +func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error { var err error // already disabled, noop unless purge @@ -137,7 +137,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -func EnableItem(hub *csconfig.Hub, target *Item) error { +func EnableItem(hub *csconfig.HubCfg, target *Item) error { var err error parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/") diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index a989ee3eedf..4cbceb4135c 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -66,7 +66,7 @@ type Walker struct { installdir string } -func NewWalker(hub *csconfig.Hub) Walker { +func NewWalker(hub *csconfig.HubCfg) Walker { return Walker{ hubdir: hub.HubDir, installdir: hub.InstallDir, @@ -404,7 +404,7 @@ func CollecDepsCheck(v *Item) error { return nil } -func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { +func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { warnings := []string{} // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last @@ -445,7 +445,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { } // Updates the info from HubInit() with the local state -func LocalSync(hub *csconfig.Hub) ([]string, error) { +func LocalSync(hub *csconfig.HubCfg) ([]string, error) { hubIdx.skippedLocal = 0 hubIdx.skippedTainted = 0 @@ -462,7 +462,7 @@ func LocalSync(hub *csconfig.Hub) ([]string, error) { return warnings, nil } -func GetHubIdx(hub *csconfig.Hub) error { +func GetHubIdx(hub *csconfig.HubCfg) error { if hub == nil { return fmt.Errorf("no configuration found for hub") } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 1abf466501b..25a89d8801d 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -39,7 +39,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string - RuntimeHubConfig *csconfig.Hub + RuntimeHubConfig *csconfig.HubCfg ResultsPath string ParserResultFile string @@ -117,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName), - RuntimeHubConfig: &csconfig.Hub{ + RuntimeHubConfig: &csconfig.HubCfg{ HubDir: runtimeHubFolder, HubIndexFile: hubTest.HubIndexFile, InstallDir: runtimeFolder, diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 53ab46b2a3c..4b603400955 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -52,10 +52,6 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error return err } - if err := csConfig.LoadHub(); err != nil { - return fmt.Errorf("loading hub: %w", err) - } - cwhub.SetHubBranch() if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 2e38e0e6c8b..7bf27670b02 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -55,16 +55,16 @@ teardown() { assert_stderr --partial "unable to create database client: unknown database type 'meh'" } -@test "crowdsec - bad configuration (empty/missing common section)" { +@test "crowdsec - default logging configuration (empty/missing common section)" { config_set '.common={}' - rune -1 "${CROWDSEC}" + rune -124 timeout 1s "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" + assert_stderr --partial "Starting processing data" config_set 'del(.common)' - rune -1 "${CROWDSEC}" + rune -124 timeout 1s "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" + assert_stderr --partial "Starting processing data" } @test "CS_LAPI_SECRET not strong enough" { From 57d3ebba1283fa75df3e5eeb25ec716e4c016223 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:03:02 +0200 Subject: [PATCH 099/263] typo (#2556) --- pkg/csconfig/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index a13ce245c51..4a544729a0f 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -92,7 +92,7 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool return &cfg, configData, nil } -// XXX: We must not not have a different behavior with an empty vs a missing configuration file. +// XXX: We must not have a different behavior with an empty vs a missing configuration file. // XXX: For this reason, all defaults have to come from NewConfig(). The following function should // XXX: be replaced func NewDefaultConfig() *Config { From 511468b8feccf5be138af19fa701cba8489706f0 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 18 Oct 2023 13:42:56 +0200 Subject: [PATCH 100/263] up --- pkg/acquisition/modules/waap/waap.go | 6 ++++++ pkg/waf/waap.go | 29 ++++++++++++++++++++++++++++ pkg/waf/waap_rule.go | 14 +++++--------- pkg/waf/waap_rule_test.go | 19 ++++++++++++++++++ pkg/waf/waap_rules_collection.go | 1 + 5 files changed, 60 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 5e7c3e632b8..d6859fddbed 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -149,6 +149,12 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { return fmt.Errorf("no waap_config provided") } + err = w.WaapRuntime.ProcessOnLoadRules() + + if err != nil { + return fmt.Errorf("unable to process on load rules : %s", err) + } + w.WaapRunners = make([]WaapRunner, w.config.Routines) for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 4100c760f42..0bafff1a80d 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -186,6 +186,35 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { return ret, nil } +func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { + for _, rule := range w.CompiledOnMatch { + if rule.FilterExpr != nil { + output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, ParsedRequest{})) + if err != nil { + return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Infof("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + for _, applyExpr := range rule.ApplyExpr { + _, err := expr.Run(applyExpr, GetHookEnv(w, ParsedRequest{})) + if err != nil { + log.Errorf("unable to apply filter: %s", err) + continue + } + } + } + return nil +} + func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { for _, rule := range w.CompiledOnMatch { diff --git a/pkg/waf/waap_rule.go b/pkg/waf/waap_rule.go index 5cc934a45f1..eabee586682 100644 --- a/pkg/waf/waap_rule.go +++ b/pkg/waf/waap_rule.go @@ -3,6 +3,7 @@ package waf import ( "fmt" "strings" + "time" ) type VPatchRule struct { @@ -17,8 +18,6 @@ type VPatchRule struct { Detect string `yaml:"detect"` //@detectXSS, @detectSQLi, etc Logic string `yaml:"logic,omitempty"` // "AND", "OR", or empty if not applicable SubRules []VPatchRule `yaml:"sub_rules,omitempty"` - - id int } func (v *VPatchRule) String() string { @@ -46,26 +45,21 @@ func (v *VPatchRule) constructRule(depth int) string { switch v.Logic { case "AND": - // Add "chain" to the current rule result = strings.TrimSuffix(result, `"`) + `,chain"` + "\n" for _, subRule := range v.SubRules { result += subRule.constructRule(depth + 1) } case "OR": skips := countTotalRules(v.SubRules) - 1 - // If the "OR" rule is at the top level and is followed by any rule, we need to count that too if depth == 0 { - skips++ // For the current rule + skips++ } - // Add the skip directive to the current rule too result = strings.TrimSuffix(result, `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) for _, subRule := range v.SubRules { skips-- if skips > 0 { - // Append skip directive and decrease the skip count result += strings.TrimSuffix(subRule.singleRuleString(), `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) } else { - // If no skip is required, append only a newline result += subRule.singleRuleString() + "\n" } } @@ -91,7 +85,9 @@ func (v *VPatchRule) singleRuleString() string { ruleStr = fmt.Sprintf(`SecRule %s "%s"`, v.Target, operator) } - actions := fmt.Sprintf(` "id:%d,deny,log`, v.id) + //FIXME: phase2 should probably not be hardcoded + //Find a better way than using time.Now().UnixMilli() to generate a unique ID + actions := fmt.Sprintf(` "id:%d,deny,log,phase:2`, time.Now().UnixNano()) // Handle transformation if v.Transform != "" { diff --git a/pkg/waf/waap_rule_test.go b/pkg/waf/waap_rule_test.go index 59cc7f89773..fcb45a2d39a 100644 --- a/pkg/waf/waap_rule_test.go +++ b/pkg/waf/waap_rule_test.go @@ -33,6 +33,25 @@ func TestVPatchRuleString(t *testing.T) { }, }, expected: `SecRule ARGS:bar "@rx [0-9]" "id:0,deny,log,chain" +SecRule REQUEST_URI "@rx /joomla/index.php/component/users/" "id:0,deny,log"`, + }, + { + name: "AND Logic Rule", + rule: VPatchRule{ + Logic: "AND", + SubRules: []VPatchRule{ + { + Target: "REQUEST_URI", + Match: "/joomla/index.php/component/users/", + }, + { + Target: "ARGS", + Variable: "bar", + Match: "[0-9]", + }, + }, + }, + expected: `SecRule ARGS:bar "@rx [0-9]" "id:0,deny,log,chain" SecRule REQUEST_URI "@rx /joomla/index.php/component/users/" "id:0,deny,log"`, }, { diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 0e3cc9eddf5..2ee094a0c26 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -27,6 +27,7 @@ type WaapCollectionConfig struct { SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` Rules []VPatchRule `yaml:"rules"` + Data interface{} `yaml:"data"` //Ignore it } func LoadCollection(collection string) (WaapCollection, error) { From 98fb84d3e7ae36ce7f73514a8aa465f904651e87 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 18 Oct 2023 17:11:43 +0200 Subject: [PATCH 101/263] be consistent : waap-rules --- cmd/crowdsec-cli/utils.go | 14 ++--- .../{waf_rules.go => waap_rules.go} | 34 ++++++------ pkg/acquisition/modules/waap/README.md | 53 +++++++++++++------ pkg/acquisition/modules/waap/waap_runner.go | 2 +- pkg/cwhub/cwhub.go | 12 ++--- pkg/cwhub/loader.go | 4 +- pkg/waf/waap_rules_collection.go | 8 +-- 7 files changed, 75 insertions(+), 52 deletions(-) rename cmd/crowdsec-cli/{waf_rules.go => waap_rules.go} (83%) diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index a071fb5f263..8f8047b564b 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -13,12 +13,12 @@ import ( "strings" "time" + "github.com/agext/levenshtein" "github.com/fatih/color" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/agext/levenshtein" "golang.org/x/exp/slices" "gopkg.in/yaml.v2" @@ -131,7 +131,7 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st items, err = cwhub.GetInstalledPostOverflowsAsString() case cwhub.COLLECTIONS: items, err = cwhub.GetInstalledCollectionsAsString() - case cwhub.WAF_RULES: + case cwhub.WAAP_RULES: items, err = cwhub.GetInstalledWafRulesAsString() default: return nil, cobra.ShellCompDirectiveDefault @@ -326,7 +326,7 @@ func ShowMetrics(hubItem *cwhub.Item) { } ShowMetrics(hubItem) } - case cwhub.WAF_RULES: + case cwhub.WAAP_RULES: log.Fatalf("FIXME: not implemented yet") default: log.Errorf("item of type '%s' is unknown", hubItem.Type) @@ -692,10 +692,10 @@ var ranges = []unit{ {value: 1e18, symbol: "E"}, {value: 1e15, symbol: "P"}, {value: 1e12, symbol: "T"}, - {value: 1e9, symbol: "G"}, - {value: 1e6, symbol: "M"}, - {value: 1e3, symbol: "k"}, - {value: 1, symbol: ""}, + {value: 1e9, symbol: "G"}, + {value: 1e6, symbol: "M"}, + {value: 1e3, symbol: "k"}, + {value: 1, symbol: ""}, } func formatNumber(num int) string { diff --git a/cmd/crowdsec-cli/waf_rules.go b/cmd/crowdsec-cli/waap_rules.go similarity index 83% rename from cmd/crowdsec-cli/waf_rules.go rename to cmd/crowdsec-cli/waap_rules.go index dcfbe009e2a..24f221c52a3 100644 --- a/cmd/crowdsec-cli/waf_rules.go +++ b/cmd/crowdsec-cli/waap_rules.go @@ -63,23 +63,23 @@ func NewWafRulesInstallCmd() *cobra.Command { var cmdWafRulesInstall = &cobra.Command{ Use: "install [config]", - Short: "Install given waf-rule(s)", - Long: `Fetch and install given waf-rule(s) from hub`, - Example: `cscli waf-rules install crowdsec/xxx crowdsec/xyz`, + Short: "Install given waap-rule(s)", + Long: `Fetch and install given waap-rule(s) from hub`, + Example: `cscli waap-rules install crowdsec/xxx crowdsec/xyz`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.WAF_RULES, args, toComplete) + return compAllItems(cwhub.WAAP_RULES, args, toComplete) }, Run: func(cmd *cobra.Command, args []string) { for _, name := range args { - t := cwhub.GetItem(cwhub.WAF_RULES, name) + t := cwhub.GetItem(cwhub.WAAP_RULES, name) if t == nil { - nearestItem, score := GetDistance(cwhub.WAF_RULES, name) - Suggest(cwhub.WAF_RULES, name, nearestItem.Name, score, ignoreError) + nearestItem, score := GetDistance(cwhub.WAAP_RULES, name) + Suggest(cwhub.WAAP_RULES, name, nearestItem.Name, score, ignoreError) continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.WAF_RULES, forceAction, downloadOnly); err != nil { + if err := cwhub.InstallItem(csConfig, name, cwhub.WAAP_RULES, forceAction, downloadOnly); err != nil { if ignoreError { log.Errorf("Error while installing '%s': %s", name, err) } else { @@ -105,11 +105,11 @@ func NewWafRulesRemoveCmd() *cobra.Command { Example: `cscli waf-rules remove crowdsec/xxx crowdsec/xyz`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, Run: func(cmd *cobra.Command, args []string) { if all { - cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, "", all, purge, forceAction) + cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, "", all, purge, forceAction) return } @@ -118,7 +118,7 @@ func NewWafRulesRemoveCmd() *cobra.Command { } for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, name, all, purge, forceAction) + cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, name, all, purge, forceAction) } }, } @@ -137,17 +137,17 @@ func NewWafRulesUpgradeCmd() *cobra.Command { Example: `cscli waf-rules upgrade crowdsec/xxx crowdsec/xyz`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, Run: func(cmd *cobra.Command, args []string) { if all { - cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, "", forceAction) + cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, "", forceAction) } else { if len(args) == 0 { log.Fatalf("no target waf rule to upgrade") } for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, name, forceAction) + cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, name, forceAction) } } }, @@ -167,10 +167,10 @@ func NewWafRulesInspectCmd() *cobra.Command { DisableAutoGenTag: true, Args: cobra.MinimumNArgs(1), ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAF_RULES, args, toComplete) + return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.WAF_RULES) + InspectItem(args[0], cwhub.WAAP_RULES) }, } cmdWafRulesInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") @@ -187,7 +187,7 @@ func NewWafRulesListCmd() *cobra.Command { cscli waf-rules list crowdsecurity/xxx`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.WAF_RULES}, args, false, true, all) + ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all) }, } cmdWafRulesList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") diff --git a/pkg/acquisition/modules/waap/README.md b/pkg/acquisition/modules/waap/README.md index 9899b137257..d4400590f92 100644 --- a/pkg/acquisition/modules/waap/README.md +++ b/pkg/acquisition/modules/waap/README.md @@ -16,7 +16,7 @@ source: waf labels: type: waf #routines: 1 -waap_config: mytest +waap_config_path: config/waap_configs/mytest.yaml ``` ## Waap config @@ -26,21 +26,20 @@ The waap config defines what rules that will be loaded by a given waap engine (a > `config/waap_configs/mytest.yaml` ```yaml -name: mytest.yaml +name: default outofband_rules: - - crowdsec/crs-default +# - crowdsecurity/crs-waf inband_rules: - - crowdsec/vpatch-default -default_remediation: block -variables_tracking: - - session_* -# onload: -# - apply: -# - DisabledInBandRuleByID(1003) -# pre_eval: -# - filter: evt.SourceIP == '1.3.4.5' -# apply: -# - DisableOutOfBandRuleByID(2302) +# - crowdsecurity/crs-waf + - crowdsecurity/custom-waf-rule +default_remediation: ban +#on_load: +# - apply: +# - SetInBand() +pre_eval: + - filter: ClientIP != '127.0.0.1' + apply: + - SetAction("ban") ``` # Waap Rules @@ -48,5 +47,27 @@ variables_tracking: For the above two to work, we need to have the two refered waap collection installed : `crowdsec/crs-default` and `crowdsec/vpatch-default`. You need to set hub_branch to ... ```yaml -cscli waf-rules install ... -``` \ No newline at end of file +type: waap-rule +name: crowdsecurity/custom-waf-rule +seclang_rules: + - SecRule ARGS:ip ";" "t:none,phase:1,log,deny,msg:'semi colon test',id:2" + +#$_GET['bar'] matches [0-9]+ AND REQUEST_URI == "/joomla/index.php/component/users/" +#REQUEST_URI == /webui/create_user AND $_POST[username] == "cisco_tac_admin" +rules: + - target: ARGS + var: bar + match: "[0-9]+" + logic: AND + sub_rules: + - target: "REQUEST_URI" + match: /joomla/index.php/component/users/ + - target: "REQUEST_URI" + equals: /webui/create_user + logic: AND + sub_rules: + - target: ARGS_POST + var: username + equals: cisco_tac_admin +``` + diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index e20fd918f1d..38a71fbc18d 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -97,7 +97,7 @@ func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *wa in = request.Tx.ProcessRequestHeaders() if in != nil { - r.logger.Infof("inband rules matched for headers : %d", in.Action) + r.logger.Infof("inband rules matched for headers : %s", in.Action) return nil } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 610e90ee6ee..40c11058b24 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -20,8 +20,8 @@ var PARSERS = "parsers" var PARSERS_OVFLW = "postoverflows" var SCENARIOS = "scenarios" var COLLECTIONS = "collections" -var WAF_RULES = "waf-rules" -var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, WAF_RULES} +var WAAP_RULES = "waap-rules" +var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, WAAP_RULES} var hubIdx map[string]map[string]Item @@ -78,7 +78,7 @@ type Item struct { PostOverflows []string `yaml:"postoverflows,omitempty" json:"postoverflows,omitempty"` Scenarios []string `yaml:"scenarios,omitempty" json:"scenarios,omitempty"` Collections []string `yaml:"collections,omitempty" json:"collections,omitempty"` - WafRules []string `yaml:"waf-rules,omitempty" json:"waf-rules,omitempty"` + WafRules []string `yaml:"waap-rules,omitempty" json:"waap-rules,omitempty"` } func (i *Item) toHubStatus() ItemHubStatus { @@ -202,7 +202,7 @@ func AddItem(itemType string, item Item) error { func DisplaySummary() { log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d waf rules", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[WAF_RULES])) + len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[WAAP_RULES])) if skippedLocal > 0 || skippedTainted > 0 { log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted) } @@ -347,10 +347,10 @@ func GetInstalledCollections() ([]Item, error) { func GetInstalledWafRules() ([]Item, error) { var retItems []Item - if _, ok := hubIdx[WAF_RULES]; !ok { + if _, ok := hubIdx[WAAP_RULES]; !ok { return nil, fmt.Errorf("no waf rules in hubIdx") } - for _, item := range hubIdx[WAF_RULES] { + for _, item := range hubIdx[WAAP_RULES] { if item.Installed { retItems = append(retItems, item) } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index ecb5d0a3a7c..e088f9338e7 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -91,8 +91,8 @@ func parser_visit(path string, f os.DirEntry, err error) error { } else if stage == COLLECTIONS { ftype = COLLECTIONS stage = "" - } else if stage == WAF_RULES { - ftype = WAF_RULES + } else if stage == WAAP_RULES { + ftype = WAAP_RULES stage = "" } else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ { return fmt.Errorf("unknown configuration type %s for file '%s'", ftype, path) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 2ee094a0c26..b02b97f5356 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -20,6 +20,8 @@ type WaapCollection struct { Rules []string } +var WAAP_RULE = "waap-rule" + // to be filled w/ seb update type WaapCollectionConfig struct { Type string `yaml:"type"` @@ -35,7 +37,7 @@ func LoadCollection(collection string) (WaapCollection, error) { //FIXME: do it once globally waapRules := make(map[string]WaapCollectionConfig) - for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAF_RULES) { + for _, hubWafRuleItem := range cwhub.GetItemMap(cwhub.WAAP_RULES) { log.Infof("loading %s", hubWafRuleItem.LocalPath) if !hubWafRuleItem.Installed { continue @@ -57,8 +59,8 @@ func LoadCollection(collection string) (WaapCollection, error) { continue } - if rule.Type != "waf-rule" { //FIXME: rename to waap-rule when hub is properly updated - log.Warnf("unexpected type %s instead of waap-rule for file %s", rule.Type, hubWafRuleItem.LocalPath) + if rule.Type != WAAP_RULE { //FIXME: rename to waap-rule when hub is properly updated + log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubWafRuleItem.LocalPath) continue } log.Infof("Adding %s to waap rules", rule.Name) From c89b42939e36d3fe33d9fa048cbc3be47482b38f Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 18 Oct 2023 17:17:57 +0200 Subject: [PATCH 102/263] naming --- cmd/crowdsec-cli/waap_rules.go | 24 ++++++++++++------------ pkg/acquisition/modules/waap/README.md | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/crowdsec-cli/waap_rules.go b/cmd/crowdsec-cli/waap_rules.go index 24f221c52a3..ad448b7abfa 100644 --- a/cmd/crowdsec-cli/waap_rules.go +++ b/cmd/crowdsec-cli/waap_rules.go @@ -12,16 +12,16 @@ import ( func NewWafRulesCmd() *cobra.Command { var cmdWafRules = &cobra.Command{ - Use: "waf-rules [action] [config]", + Use: "waap-rules [action] [config]", Short: "Install/Remove/Upgrade/Inspect waf-rule(s) from hub", - Example: `cscli waf-rules install crowdsecurity/core-rule-set -cscli waf-rules inspect crowdsecurity/core-rule-set -cscli waf-rules upgrade crowdsecurity/core-rule-set -cscli waf-rules list -cscli waf-rules remove crowdsecurity/core-rule-set + Example: `cscli waap-rules install crowdsecurity/core-rule-set +cscli waap-rules inspect crowdsecurity/core-rule-set +cscli waap-rules upgrade crowdsecurity/core-rule-set +cscli waap-rules list +cscli waap-rules remove crowdsecurity/core-rule-set `, Args: cobra.MinimumNArgs(1), - Aliases: []string{"waf-rule"}, + Aliases: []string{"waap-rule"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadHub(); err != nil { @@ -102,7 +102,7 @@ func NewWafRulesRemoveCmd() *cobra.Command { Short: "Remove given waf-rule(s)", Long: `Remove given waf-rule(s) from hub`, Aliases: []string{"delete"}, - Example: `cscli waf-rules remove crowdsec/xxx crowdsec/xyz`, + Example: `cscli waap-rules remove crowdsec/xxx crowdsec/xyz`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) @@ -134,7 +134,7 @@ func NewWafRulesUpgradeCmd() *cobra.Command { Use: "upgrade [config]", Short: "Upgrade given waf-rule(s)", Long: `Fetch and upgrade given waf-rule(s) from hub`, - Example: `cscli waf-rules upgrade crowdsec/xxx crowdsec/xyz`, + Example: `cscli waap-rules upgrade crowdsec/xxx crowdsec/xyz`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) @@ -163,7 +163,7 @@ func NewWafRulesInspectCmd() *cobra.Command { Use: "inspect [name]", Short: "Inspect given waf rule", Long: `Inspect given waf rule`, - Example: `cscli waf-rules inspect crowdsec/xxx`, + Example: `cscli waap-rules inspect crowdsec/xxx`, DisableAutoGenTag: true, Args: cobra.MinimumNArgs(1), ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -183,8 +183,8 @@ func NewWafRulesListCmd() *cobra.Command { Use: "list [name]", Short: "List all waf rules or given one", Long: `List all waf rules or given one`, - Example: `cscli waf-rules list -cscli waf-rules list crowdsecurity/xxx`, + Example: `cscli waap-rules list +cscli waap-rules list crowdsecurity/xxx`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all) diff --git a/pkg/acquisition/modules/waap/README.md b/pkg/acquisition/modules/waap/README.md index d4400590f92..d2d4b23191f 100644 --- a/pkg/acquisition/modules/waap/README.md +++ b/pkg/acquisition/modules/waap/README.md @@ -16,14 +16,14 @@ source: waf labels: type: waf #routines: 1 -waap_config_path: config/waap_configs/mytest.yaml +waap_config_path: config/waap-configs/mytest.yaml ``` ## Waap config The waap config defines what rules that will be loaded by a given waap engine (associated with an acquis). -> `config/waap_configs/mytest.yaml` +> `config/waap-configs/mytest.yaml` ```yaml name: default From 2600ffbd1965df10b88e5a677ff7858a233eb300 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 19 Oct 2023 10:25:55 +0200 Subject: [PATCH 103/263] delete coraza submodule --- .gitmodules | 4 ---- coraza | 1 - 2 files changed, 5 deletions(-) delete mode 160000 coraza diff --git a/.gitmodules b/.gitmodules index f3e8a2f5f88..6a170946b4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,7 +14,3 @@ [submodule "tests/lib/bats-mock"] path = test/lib/bats-mock url = https://github.com/crowdsecurity/bats-mock.git -[submodule "coraza"] - path = coraza - url = http://github.com/buixor/coraza - branch = testing diff --git a/coraza b/coraza deleted file mode 160000 index 063bbc322dc..00000000000 --- a/coraza +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 063bbc322dc2ad389c354e7703638104412e197a From 88e4f7c15755321230adb8c190ac39cf77b3dc22 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 19 Oct 2023 12:04:29 +0200 Subject: [PATCH 104/263] Refact pkg/csconfig, pkg/cwhub (#2555) * csconfig: drop redundant hub information on *Cfg structs * rename validItemFileName() -> item.validPath() * Methods on hub object * updated tests to reduce need of csconfig.Config or global state --- cmd/crowdsec-cli/capi.go | 5 +- cmd/crowdsec-cli/collections.go | 31 +++- cmd/crowdsec-cli/config_backup.go | 13 +- cmd/crowdsec-cli/config_restore.go | 17 +- cmd/crowdsec-cli/config_show.go | 1 - cmd/crowdsec-cli/console.go | 5 +- cmd/crowdsec-cli/hub.go | 29 ++-- cmd/crowdsec-cli/item_metrics.go | 6 +- cmd/crowdsec-cli/item_suggest.go | 18 ++- cmd/crowdsec-cli/items.go | 32 ++-- cmd/crowdsec-cli/lapi.go | 11 +- cmd/crowdsec-cli/parsers.go | 29 +++- cmd/crowdsec-cli/postoverflows.go | 29 +++- cmd/crowdsec-cli/require/require.go | 11 +- cmd/crowdsec-cli/scenarios.go | 29 +++- cmd/crowdsec-cli/simulation.go | 5 +- cmd/crowdsec-cli/support.go | 13 +- cmd/crowdsec-cli/utils_table.go | 4 +- cmd/crowdsec/crowdsec.go | 6 +- cmd/crowdsec/main.go | 6 +- cmd/crowdsec/output.go | 9 +- pkg/csconfig/config.go | 2 +- pkg/csconfig/crowdsec_service.go | 11 +- pkg/csconfig/crowdsec_service_test.go | 15 -- pkg/csconfig/cscli.go | 11 +- pkg/csconfig/cscli_test.go | 4 - pkg/csconfig/hub.go | 3 + pkg/cwhub/cwhub.go | 44 ++--- pkg/cwhub/cwhub_test.go | 225 +++++++++++++------------- pkg/cwhub/download.go | 70 ++++---- pkg/cwhub/download_test.go | 18 ++- pkg/cwhub/helpers.go | 39 +++-- pkg/cwhub/helpers_test.go | 166 +++++++++---------- pkg/cwhub/{hubindex.go => hub.go} | 40 +++-- pkg/cwhub/install.go | 36 ++--- pkg/cwhub/loader.go | 118 +++++++------- pkg/hubtest/hubtest.go | 4 +- pkg/hubtest/hubtest_item.go | 18 +-- pkg/leakybucket/buckets_test.go | 32 +++- pkg/leakybucket/manager_load.go | 11 +- pkg/leakybucket/tests/hub/index.json | 1 + pkg/parser/unix_parser.go | 14 +- pkg/setup/install.go | 11 +- 43 files changed, 663 insertions(+), 539 deletions(-) rename pkg/cwhub/{hubindex.go => hub.go} (76%) create mode 100644 pkg/leakybucket/tests/hub/index.json diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 0e0f217aa5f..3f94d286c84 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command { return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 158b808234a..494cedab27e 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -25,7 +25,7 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables Aliases: []string{"collection"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.COLLECTIONS, name) + t := hub.GetItem(cwhub.COLLECTIONS, name) if t == nil { nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force) + err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force) if err != nil { return err } @@ -141,7 +151,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { for _, name := range args { if !force { - item := cwhub.GetItem(cwhub.COLLECTIONS, name) + item := hub.GetItem(cwhub.COLLECTIONS, name) if item == nil { // XXX: this should be in GetItem? return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) @@ -153,7 +163,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { } } - err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force) + err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force) if err != nil { return err } @@ -197,8 +207,13 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { return err } return nil @@ -209,7 +224,7 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 73d1aca03db..417b21eda31 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) -func backupHub(dirPath string) error { +func backupHub(hub *cwhub.Hub, dirPath string) error { var err error var itemDirectory string var upstreamParsers []string @@ -22,7 +22,7 @@ func backupHub(dirPath string) error { clog := log.WithFields(log.Fields{ "type": itemType, }) - itemMap := cwhub.GetItemMap(itemType) + itemMap := hub.GetItemMap(itemType) if itemMap == nil { clog.Infof("No %s to backup.", itemType) continue @@ -189,7 +189,12 @@ func backupConfigToDirectory(dirPath string) error { log.Infof("Saved profiles to %s", backupProfiles) } - if err = backupHub(dirPath); err != nil { + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if err = backupHub(hub, dirPath); err != nil { return fmt.Errorf("failed to backup hub config: %s", err) } @@ -197,7 +202,7 @@ func backupConfigToDirectory(dirPath string) error { } func runConfigBackup(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index ccd9ebc5ed4..ba734461eb1 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -23,23 +23,28 @@ type OldAPICfg struct { // it's a rip of the cli version, but in silent-mode func silentInstallItem(name string, obtype string) (string, error) { - var item = cwhub.GetItem(obtype, name) + hub, err := cwhub.GetHub() + if err != nil { + return "", err + } + + var item = hub.GetItem(obtype, name) if item == nil { return "", fmt.Errorf("error retrieving item") } - err := cwhub.DownloadLatest(csConfig.Hub, item, false, false) + err = hub.DownloadLatest(item, false, false) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } - if err := cwhub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(obtype, *item); err != nil { return "", err } - err = cwhub.EnableItem(csConfig.Hub, item) + err = hub.EnableItem(item) if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) } - if err := cwhub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(obtype, *item); err != nil { return "", err } return fmt.Sprintf("Enabled %s", item.Name), nil @@ -292,7 +297,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index 9f5b11fc190..ca70511955e 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled cscli: - Output : {{.Cscli.Output}} - Hub Branch : {{.Cscli.HubBranch}} - - Hub Folder : {{.Cscli.HubDir}} {{- end }} {{- if .API }} diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 439a8143b1e..06eae5e56ee 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba return fmt.Errorf("could not parse CAPI URL: %s", err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get installed scenarios: %s", err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 501fbc498b9..c4ec140e94e 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -50,12 +50,13 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - if err = require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) + warn, _ := hub.LocalSync() for _, v := range warn { log.Info(v) } @@ -88,19 +89,24 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + cwhub.SetHubBranch() + + // don't use require.Hub because if there is no index file, it would fail + + hub, err := cwhub.InitHubUpdate(csConfig.Hub) + if err != nil { if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) } log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) cwhub.HubBranch = "master" - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) + warn, _ := hub.LocalSync() for _, v := range warn { log.Info(v) } @@ -122,8 +128,6 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - return nil }, RunE: runHubUpdate, @@ -140,27 +144,28 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } log.Infof("Upgrading collections") - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { return err } log.Infof("Upgrading parsers") - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { return err } log.Infof("Upgrading scenarios") - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { return err } log.Infof("Upgrading postoverflows") - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { return err } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index ec925262e9d..924139b93a8 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -18,7 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func ShowMetrics(hubItem *cwhub.Item) { +func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) @@ -36,11 +36,11 @@ func ShowMetrics(hubItem *cwhub.Item) { scenarioMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) + hubItem = hub.GetItem(cwhub.COLLECTIONS, item) if hubItem == nil { log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) } - ShowMetrics(hubItem) + ShowMetrics(hub, hubItem) } default: log.Errorf("item of type '%s' is unknown", hubItem.Type) diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index c52239fb478..2b48cdc1231 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -33,7 +33,11 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { allItems := make([]string, 0) nearestScore := 100 nearestItem := &cwhub.Item{} - hubItems := cwhub.GetItemMap(itemType) + + // XXX: handle error + hub, _ := cwhub.GetHub() + + hubItems := hub.GetItemMap(itemType) for _, item := range hubItems { allItems = append(allItems, item.Name) } @@ -42,19 +46,20 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { d := levenshtein.Distance(itemName, s, nil) if d < nearestScore { nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) + nearestItem = hub.GetItem(itemType, s) } } return nearestItem, nearestScore } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return nil, cobra.ShellCompDirectiveDefault } comp := make([]string, 0) - hubItems := cwhub.GetItemMap(itemType) + hubItems := hub.GetItemMap(itemType) for _, item := range hubItems { if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { comp = append(comp, item.Name) @@ -65,11 +70,12 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string, } func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return nil, cobra.ShellCompDirectiveDefault } - items, err := cwhub.GetInstalledItemsAsString(itemType) + items, err := hub.GetInstalledItemsAsString(itemType) if err != nil { cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) return nil, cobra.ShellCompDirectiveDefault diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index cebf5b723d6..371cdc16d4f 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -16,8 +16,8 @@ import ( ) -func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) { - itemNames := cwhub.GetItemNames(itemType) +func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) { + itemNames := hub.GetItemNames(itemType) notExist := []string{} if len(args) > 0 { @@ -40,7 +40,7 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string, if installedOnly { installed := []string{} for _, item := range itemNames { - if cwhub.GetItem(itemType, item).Installed { + if hub.GetItem(itemType, item).Installed { installed = append(installed, item) } } @@ -52,9 +52,15 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string, func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { var err error + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + items := make(map[string][]string) for _, itemType := range itemTypes { - if items[itemType], err = selectItems(itemType, args, !all); err != nil { + if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil { return err } } @@ -78,7 +84,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, // empty slice in case there are no items of this type hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) for i, itemName := range items[itemType] { - item := cwhub.GetItem(itemType, itemName) + item := hub.GetItem(itemType, itemName) status, emo := item.Status() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, @@ -112,7 +118,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, } for _, itemType := range itemTypes { for _, itemName := range items[itemType] { - item := cwhub.GetItem(itemType, itemName) + item := hub.GetItem(itemType, itemName) status, _ := item.Status() if item.LocalVersion == "" { item.LocalVersion = "n/a" @@ -138,15 +144,17 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, } func InspectItem(name string, itemType string, noMetrics bool) error { - hubItem := cwhub.GetItem(itemType, name) + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + hubItem := hub.GetItem(itemType, name) if hubItem == nil { return fmt.Errorf("can't find '%s' in %s", name, itemType) } - var ( - b []byte - err error - ) + var b []byte switch csConfig.Cscli.Output { case "human", "raw": @@ -168,7 +176,7 @@ func InspectItem(name string, itemType string, noMetrics bool) error { } fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hubItem) + ShowMetrics(hub, hubItem) return nil } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 37ee0088c9e..aa43a3d6d83 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { log.Fatalf("parsing api url ('%s'): %s", apiurl, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Fatal(err) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { log.Fatalf("failed to get scenarios : %s", err) } @@ -338,9 +339,9 @@ cscli lapi context detect crowdsecurity/sshd-logs log.Fatalf("Failed to init expr helpers : %s", err) } - // Populate cwhub package tools - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Fatalf("Failed to load hub index : %s", err) + _, err = require.Hub(csConfig) + if err != nil { + log.Fatal(err) } csParsers := parser.NewParsers() diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index 49e301a6c7c..e0d294e0191 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -25,7 +25,7 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs Aliases: []string{"parser"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runParsersInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS, name) + t := hub.GetItem(cwhub.PARSERS, name) if t == nil { nearestItem, score := GetDistance(cwhub.PARSERS, name) Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runParsersInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runParsersRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force) + err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runParsersRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force) + err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index 8c5f1b1754e..a73bf3da450 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -25,7 +25,7 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns Aliases: []string{"postoverflow"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name) + t := hub.GetItem(cwhub.POSTOVERFLOWS, name) if t == nil { nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force) + err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force) + err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index fba59cb1e40..b477e9e00c9 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -64,16 +64,17 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub (c *csconfig.Config) error { +func Hub (c *csconfig.Config) (*cwhub.Hub, error) { if c.Hub == nil { - return fmt.Errorf("you must configure cli before interacting with hub") + return nil, fmt.Errorf("you must configure cli before interacting with hub") } cwhub.SetHubBranch() - if err := cwhub.GetHubIdx(c.Hub); err != nil { - return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) + hub, err := cwhub.InitHub(c.Hub) + if err != nil { + return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) } - return nil + return hub, nil } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 863e50236ba..23df3f5ff01 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -25,7 +25,7 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing Aliases: []string{"scenario"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.SCENARIOS, name) + t := hub.GetItem(cwhub.SCENARIOS, name) if t == nil { nearestItem, score := GetDistance(cwhub.SCENARIOS, name) Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force) + err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force) + err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 890785a2d08..0ad4ff5aedd 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -145,13 +145,14 @@ func NewSimulationEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Fatal(err) } if len(args) > 0 { for _, scenario := range args { - var item = cwhub.GetItem(cwhub.SCENARIOS, scenario) + var item = hub.GetItem(cwhub.SCENARIOS, scenario) if item == nil { log.Errorf("'%s' doesn't exist or is not a scenario", scenario) continue diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 0c87ad3c97d..e8462c4ae11 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -155,7 +155,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) { return out.Bytes(), nil } -func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte { +func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte { if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { return []byte("No agent credentials found, are we LAPI ?") } @@ -165,7 +165,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str if err != nil { return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) } @@ -293,7 +293,8 @@ cscli support dump -f /tmp/crowdsec-support.zip skipAgent = true } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") skipHub = true infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) @@ -356,7 +357,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login, csConfig.API.Server.OnlineClient.Credentials.Password, csConfig.API.Server.OnlineClient.Credentials.URL, - CAPIURLPrefix) + CAPIURLPrefix, + hub) } if !skipLAPI { @@ -364,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login, csConfig.API.Client.Credentials.Password, csConfig.API.Client.Credentials.URL, - LAPIURLPrefix) + LAPIURLPrefix, + hub) infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile() } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 840250f9c8a..1f3f62038ac 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -16,8 +16,10 @@ func listHubItemTable(out io.Writer, title string, itemType string, itemNames [] t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) + hub, _ := cwhub.GetHub() + for itemName := range itemNames { - item := cwhub.GetItem(itemType, itemNames[itemName]) + item := hub.GetItem(itemType, itemNames[itemName]) status, emo := item.Status() t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index c573cd4d445..dd81ad34564 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,8 +23,8 @@ import ( func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error - // Populate cwhub package tools - if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { + hub, err := cwhub.InitHub(cConfig.Hub) + if err != nil { return nil, fmt.Errorf("while loading hub index: %w", err) } @@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { return nil, fmt.Errorf("while loading parsers: %w", err) } - if err := LoadBuckets(cConfig); err != nil { + if err := LoadBuckets(cConfig, hub); err != nil { return nil, fmt.Errorf("while loading scenarios: %w", err) } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index e93dbef044c..d4c2ceb9095 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -75,12 +75,12 @@ type Flags struct { type labelsMap map[string]string -func LoadBuckets(cConfig *csconfig.Config) error { +func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { var ( err error files []string ) - for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) { + for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) { if hubScenarioItem.Installed { files = append(files, hubScenarioItem.LocalPath) } @@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config) error { buckets = leakybucket.NewBuckets() log.Infof("Loading %d scenario files", len(files)) - holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent) + holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { return fmt.Errorf("scenario loading failed: %v", err) diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 95642bbf31f..95235aa9ad7 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -70,7 +70,12 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky var cache []types.RuntimeAlert var cacheMutex sync.Mutex - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("loading list of installed hub scenarios: %w", err) } @@ -93,7 +98,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky URL: apiURL, PapiURL: papiURL, VersionPrefix: "v1", - UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, + UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, }) if err != nil { return fmt.Errorf("new client api: %w", err) diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 4a544729a0f..27bb32b419a 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -100,7 +100,7 @@ func NewDefaultConfig() *Config { commonCfg := CommonCfg{ Daemonize: false, LogMedia: "stdout", - LogLevel: &logLevel, + LogLevel: &logLevel, } prometheus := PrometheusCfg{ Enabled: true, diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index f9602a5be92..dc226cfd6ac 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct { BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` SimulationFilePath string `yaml:"-"` ContextToSend map[string][]string `yaml:"-"` } @@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("load error (simulation): %w", err) } - c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir - c.Crowdsec.DataDir = c.ConfigPaths.DataDir - c.Crowdsec.HubDir = c.ConfigPaths.HubDir - c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile - if c.Crowdsec.ParserRoutinesCount <= 0 { c.Crowdsec.ParserRoutinesCount = 1 } @@ -149,7 +140,7 @@ func (c *Config) LoadCrowdsec() error { fallback := false if c.Crowdsec.ConsoleContextPath == "" { // fallback to default config file - c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml") + c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") fallback = true } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index 06a7e91bd85..e9d7e8de373 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -54,11 +54,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: "", ConsoleContextPath: contextFileFullPath, AcquisitionFilePath: acquisFullPath, - ConfigDir: "./testdata", - DataDir: "./data", - HubDir: "./hub", - // XXX: need to ensure a default here - HubIndexFile: "", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -98,11 +93,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: acquisDirFullPath, AcquisitionFilePath: acquisFullPath, ConsoleContextPath: contextFileFullPath, - ConfigDir: "./testdata", - // XXX: need to ensure a default here - HubIndexFile: "", - DataDir: "./data", - HubDir: "./hub", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -139,11 +129,6 @@ func TestLoadCrowdsec(t *testing.T) { Enable: ptr.Of(true), AcquisitionDirPath: "", AcquisitionFilePath: "", - ConfigDir: "./testdata", - // XXX: need to ensure a default here - HubIndexFile: "", - DataDir: "./data", - HubDir: "./hub", ConsoleContextPath: contextFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 8db0a184872..6ecce4a53f2 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -11,10 +11,7 @@ type CscliCfg struct { HubBranch string `yaml:"hub_branch"` SimulationConfig *SimulationConfig `yaml:"-"` DbConfig *DatabaseCfg `yaml:"-"` - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` + SimulationFilePath string `yaml:"-"` PrometheusUrl string `yaml:"prometheus_uri"` } @@ -23,10 +20,8 @@ func (c *Config) loadCSCLI() error { if c.Cscli == nil { c.Cscli = &CscliCfg{} } - c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir - c.Cscli.DataDir = c.ConfigPaths.DataDir - c.Cscli.HubDir = c.ConfigPaths.HubDir - c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile + + // XXX: HubBranch default should be set here and fed to HubCfg? if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index e928eda5a63..b814fda8892 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -32,10 +32,6 @@ func TestLoadCSCLI(t *testing.T) { }, }, expected: &CscliCfg{ - ConfigDir: "./testdata", - DataDir: "./data", - HubDir: "./hub", - HubIndexFile: "./hub/.index.json", PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index 2164c19b25d..db49c0231df 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -9,6 +9,9 @@ type HubCfg struct { } func (c *Config) loadHub() error { + + // XXX: HubBranch too -- from cscli or chooseHubBranch() ? + c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 580bf4e8196..7ae80549231 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -15,7 +15,6 @@ import ( "golang.org/x/mod/semver" ) - var ( ErrMissingReference = errors.New("Reference(s) missing in collection") @@ -40,7 +39,7 @@ type Item struct { FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any // remote (hub) info @@ -112,9 +111,16 @@ func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) } +// validPath returns true if the (relative) path is allowed for the item +// dirNmae: the directory name (ie. crowdsecurity) +// fileName: the filename (ie. apache2-logs.yaml) +func (i *Item) validPath(dirName, fileName string) bool { + return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") +} + // GetItemMap returns the map of items for a given type -func GetItemMap(itemType string) map[string]Item { - m, ok := hubIdx.Items[itemType] +func (h *Hub) GetItemMap(itemType string) map[string]Item { + m, ok := h.Items[itemType] if !ok { return nil } @@ -123,6 +129,7 @@ func GetItemMap(itemType string) map[string]Item { } // itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary +// XXX: only used by leakybucket manager func itemKey(itemPath string) (string, error) { f, err := os.Lstat(itemPath) if err != nil { @@ -150,13 +157,13 @@ func itemKey(itemPath string) (string, error) { } // GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. -func GetItemByPath(itemType string, itemPath string) (*Item, error) { +func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { itemKey, err := itemKey(itemPath) if err != nil { return nil, err } - m := GetItemMap(itemType) + m := h.GetItemMap(itemType) if m == nil { return nil, fmt.Errorf("item type %s doesn't exist", itemType) } @@ -170,19 +177,20 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { } // GetItem returns the item from hub based on its type and full name (author/name) -func GetItem(itemType string, itemName string) *Item { - if m, ok := GetItemMap(itemType)[itemName]; ok { - return &m +func (h *Hub) GetItem(itemType string, itemName string) *Item { + m, ok := h.GetItemMap(itemType)[itemName] + if !ok { + return nil } - return nil + return &m } // GetItemNames returns the list of item (full) names for a given type // ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx // The names can be used to retrieve the item with GetItem() -func GetItemNames(itemType string) []string { - m := GetItemMap(itemType) +func (h *Hub) GetItemNames(itemType string) []string { + m := h.GetItemMap(itemType) if m == nil { return nil } @@ -196,10 +204,10 @@ func GetItemNames(itemType string) []string { } // AddItem adds an item to the hub index -func AddItem(itemType string, item Item) error { +func (h *Hub) AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { - hubIdx.Items[itemType][item.Name] = item + h.Items[itemType][item.Name] = item return nil } } @@ -208,8 +216,8 @@ func AddItem(itemType string, item Item) error { } // GetInstalledItems returns the list of installed items -func GetInstalledItems(itemType string) ([]Item, error) { - items, ok := hubIdx.Items[itemType] +func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { + items, ok := h.Items[itemType] if !ok { return nil, fmt.Errorf("no %s in hubIdx", itemType) } @@ -226,8 +234,8 @@ func GetInstalledItems(itemType string) ([]Item, error) { } // GetInstalledItemsAsString returns the names of the installed items -func GetInstalledItemsAsString(itemType string) ([]string, error) { - items, err := GetInstalledItems(itemType) +func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { + items, err := h.GetInstalledItems(itemType) if err != nil { return nil, err } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 7f3a50a1a50..216a0df4bbd 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,23 +29,15 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) - - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") + hub := envSetup(t) // get existing map - x := GetItemMap(COLLECTIONS) + x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) // Get item : good and bad for k := range x { - item := GetItem(COLLECTIONS, k) + item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) item.Installed = true @@ -65,76 +57,101 @@ func TestItemStatus(t *testing.T) { require.Equal(t, "disabled,local", txt) } - DisplaySummary() + err := DisplaySummary() + require.NoError(t, err) } func TestGetters(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) - - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") + hub := envSetup(t) // get non existing map - empty := GetItemMap("ratata") + empty := hub.GetItemMap("ratata") require.Nil(t, empty) // get existing map - x := GetItemMap(COLLECTIONS) + x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) // Get item : good and bad for k := range x { - empty := GetItem(COLLECTIONS, k+"nope") + empty := hub.GetItem(COLLECTIONS, k+"nope") require.Nil(t, empty) - item := GetItem(COLLECTIONS, k) + item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) // Add item and get it item.Name += "nope" - err := AddItem(COLLECTIONS, *item) + err := hub.AddItem(COLLECTIONS, *item) require.NoError(t, err) - newitem := GetItem(COLLECTIONS, item.Name) + newitem := hub.GetItem(COLLECTIONS, item.Name) require.NotNil(t, newitem) - err = AddItem("ratata", *item) + err = hub.AddItem("ratata", *item) cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") } } func TestIndexDownload(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) + _, err := InitHubUpdate(hub.cfg) require.NoError(t, err, "failed to download index") - err = GetHubIdx(cfg.Hub) + _, err = GetHub() require.NoError(t, err, "failed to load hub index") } -func getTestCfg() *csconfig.Config { - cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}} - cfg.Hub.InstallDir, _ = filepath.Abs("./install") - cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") - cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") +// testHub initializes a temporary hub with an empty json file, optionally updating it +func testHub(t *testing.T, update bool) *Hub { + tmpDir, err := os.MkdirTemp("", "testhub") + require.NoError(t, err) + + hubCfg := &csconfig.HubCfg{ + HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tmpDir, "crowdsec"), + InstallDataDir: filepath.Join(tmpDir, "installed-data"), + } + + err = os.MkdirAll(hubCfg.HubDir, 0700) + require.NoError(t, err) + + err = os.MkdirAll(hubCfg.InstallDir, 0700) + require.NoError(t, err) + + err = os.MkdirAll(hubCfg.InstallDataDir, 0700) + require.NoError(t, err) + + index, err := os.Create(hubCfg.HubIndexFile) + require.NoError(t, err) + + _, err = index.WriteString(`{}`) + require.NoError(t, err) - return cfg + index.Close() + + t.Cleanup(func() { + os.RemoveAll(tmpDir) + }) + + constructor := InitHub + + if update { + constructor = InitHubUpdate + } + + hub, err := constructor(hubCfg) + require.NoError(t, err) + + return hub } -func envSetup(t *testing.T) *csconfig.Config { +func envSetup(t *testing.T) *Hub { resetResponseByPath() log.SetLevel(log.DebugLevel) - cfg := getTestCfg() - defaultTransport := http.DefaultClient.Transport t.Cleanup(func() { @@ -144,14 +161,7 @@ func envSetup(t *testing.T) *csconfig.Config { // Mock the http client http.DefaultClient.Transport = newMockTransport() - err := os.MkdirAll(cfg.Hub.InstallDir, 0700) - require.NoError(t, err) - - err = os.MkdirAll(cfg.Hub.HubDir, 0700) - require.NoError(t, err) - - err = UpdateHubIdx(cfg.Hub) - require.NoError(t, err) + hub := testHub(t, true) // if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { // log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) @@ -159,42 +169,33 @@ func envSetup(t *testing.T) *csconfig.Config { // if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil { // log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err) // } - return cfg -} - -func envTearDown(cfg *csconfig.Config) { - if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - } - - if err := os.RemoveAll(cfg.Hub.HubDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err) - } + return hub } -func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { +func testInstallItem(hub *Hub, t *testing.T, item Item) { // Install the parser - err := DownloadLatest(cfg, &item, false, false) + + err := hub.DownloadLatest(&item, false, false) require.NoError(t, err, "failed to download %s", item.Name) - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - err = EnableItem(cfg, &item) + err = hub.EnableItem(&item) require.NoError(t, err, "failed to enable %s", item.Name) - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } -func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) +func testTaintItem(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) @@ -205,54 +206,54 @@ func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) // Local sync and check status - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } -func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) +func testUpdateItem(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := DownloadLatest(cfg, &item, true, true) + err := hub.DownloadLatest(&item, true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } -func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) +func testDisableItem(hub *Hub, t *testing.T, item Item) { + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove - err := DisableItem(cfg, &item, false, false) + err := hub.DisableItem(&item, false, false) require.NoError(t, err, "failed to disable %s", item.Name) // Local sync and check status - warns, err := LocalSync(cfg) + warns, err := hub.LocalSync() require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge - err = DisableItem(cfg, &item, true, false) + err = hub.DisableItem(&item, true, false) require.NoError(t, err, "failed to purge %s", item.Name) // Local sync and check status - warns, err = LocalSync(cfg) + warns, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { @@ -265,20 +266,18 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx.Items[PARSERS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testTaintItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testUpdateItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testDisableItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] + for _, it := range hub.Items[PARSERS] { + testInstallItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testTaintItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testUpdateItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testDisableItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] break } @@ -294,19 +293,17 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx.Items[COLLECTIONS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testTaintItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testUpdateItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testDisableItem(cfg.Hub, t, it) + for _, it := range hub.Items[COLLECTIONS] { + testInstallItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testTaintItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testUpdateItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testDisableItem(hub, t, it) break } } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 7b6771867dd..0f4a8fb7a7a 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -19,31 +19,39 @@ import ( var ErrIndexNotFound = fmt.Errorf("index not found") -// UpdateHubIdx downloads the latest version of the index and updates the one in memory -func UpdateHubIdx(hub *csconfig.HubCfg) error { - bidx, err := DownloadHubIdx(hub) +// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk +// It is used to inizialize the hub when there is no index file yet +func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") + } + + bidx, err := DownloadHubIdx(cfg.HubIndexFile) if err != nil { - return fmt.Errorf("failed to download index: %w", err) + return nil, fmt.Errorf("failed to download index: %w", err) } ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("failed to read index: %w", err) + return nil, fmt.Errorf("failed to read index: %w", err) } } - hubIdx = HubIndex{Items: ret} + theHub = &Hub{ + Items: ret, + cfg: cfg, + } - if _, err := LocalSync(hub); err != nil { - return fmt.Errorf("failed to sync: %w", err) + if _, err := theHub.LocalSync(); err != nil { + return nil, fmt.Errorf("failed to sync: %w", err) } - return nil + return theHub, nil } // DownloadHubIdx downloads the latest version of the index and returns the content -func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { +func DownloadHubIdx(indexPath string) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) @@ -70,7 +78,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } - oldContent, err := os.ReadFile(hub.HubIndexFile) + oldContent, err := os.ReadFile(indexPath) if err != nil { if !os.IsNotExist(err) { log.Warningf("failed to read hub index: %s", err) @@ -80,7 +88,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { // write it anyway, can't hurt } - file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return nil, fmt.Errorf("while opening hub index file: %w", err) @@ -92,13 +100,13 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { return nil, fmt.Errorf("while writing hub index file: %w", err) } - log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile) + log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) return body, nil } // DownloadLatest will download the latest version of Item to the tdir directory -func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error { +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -109,7 +117,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn return nil } - return DownloadItem(hub, target, overwrite) + return h.DownloadItem(target, overwrite) } // collection @@ -117,7 +125,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx.Items[ptrtype][p] + val, ok := h.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -132,7 +140,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn if ptrtype == COLLECTIONS { log.Tracef("collection, recurse") - err = DownloadLatest(hub, &val, overwrite, updateOnly) + err = h.DownloadLatest(&val, overwrite, updateOnly) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -140,7 +148,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn downloaded := val.Downloaded - err = DownloadItem(hub, &val, overwrite) + err = h.DownloadItem(&val, overwrite) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -148,16 +156,16 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { - if err = EnableItem(hub, &val); err != nil { + if err = h.EnableItem(&val); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } - hubIdx.Items[ptrtype][p] = val + h.Items[ptrtype][p] = val } } - err = DownloadItem(hub, target, overwrite) + err = h.DownloadItem(target, overwrite) if err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -165,8 +173,8 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn return nil } -func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { - tdir := hub.HubDir +func (h *Hub) DownloadItem(target *Item, overwrite bool) error { + tdir := h.cfg.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { @@ -202,12 +210,12 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { return fmt.Errorf("while reading %s: %w", req.URL.String(), err) } - h := sha256.New() - if _, err = h.Write(body); err != nil { + hash := sha256.New() + if _, err = hash.Write(body); err != nil { return fmt.Errorf("while hashing %s: %w", target.Name, err) } - meow := fmt.Sprintf("%x", h.Sum(nil)) + meow := fmt.Sprintf("%x", hash.Sum(nil)) if meow != target.Versions[target.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) @@ -263,18 +271,18 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { target.Tainted = false target.UpToDate = true - if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } // DownloadDataIfNeeded downloads the data files for an item -func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) +func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) if err != nil { @@ -283,7 +291,7 @@ func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { defer itemFile.Close() - if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil { + if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index b1b41c5792f..ed0763081df 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -2,12 +2,11 @@ package cwhub import ( "fmt" + "os" "strings" "testing" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) func TestDownloadHubIdx(t *testing.T) { @@ -15,9 +14,18 @@ func TestDownloadHubIdx(t *testing.T) { // bad url template fmt.Println("Test 'bad URL'") + tmpIndex, err := os.CreateTemp("", "index.json") + if err != nil { + t.Fatalf("failed to create temp file : %s", err) + } + + t.Cleanup(func() { + os.Remove(tmpIndex.Name()) + }) + RawFileURLTemplate = "x" - ret, err := DownloadHubIdx(&csconfig.HubCfg{}) + ret, err := DownloadHubIdx(tmpIndex.Name()) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { log.Errorf("unexpected error %s", err) } @@ -29,7 +37,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "https://baddomain/%s/%s" - ret, err = DownloadHubIdx(&csconfig.HubCfg{}) + ret, err = DownloadHubIdx(tmpIndex.Name()) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { log.Errorf("unexpected error %s", err) } @@ -41,7 +49,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = back - ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"}) + ret, err = DownloadHubIdx("/does/not/exist/index.json") if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { log.Errorf("unexpected error %s", err) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 2db76863638..05c34824d40 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -8,7 +8,6 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) @@ -62,8 +61,8 @@ func SetHubBranch() { } // InstallItem installs an item from the hub -func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error { - item := GetItem(itemType, name) +func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { + item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) } @@ -76,26 +75,26 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force } } - err := DownloadLatest(csConfig.Hub, item, force, true) + err := h.DownloadLatest(item, force, true) if err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = AddItem(itemType, *item); err != nil { + if err = h.AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } if downloadOnly { - log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath)) + log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath)) return nil } - err = EnableItem(csConfig.Hub, item) + err = h.EnableItem(item) if err != nil { return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := AddItem(itemType, *item); err != nil { + if err := h.AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -105,20 +104,20 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force } // RemoveItem removes one - or all - the items from the hub -func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error { +func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error { if name != "" { - item := GetItem(itemType, name) + item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("can't find '%s' in %s", name, itemType) } - err := DisableItem(csConfig.Hub, item, purge, forceAction) + err := h.DisableItem(item, purge, forceAction) if err != nil { return fmt.Errorf("unable to disable %s: %w", item.Name, err) } - if err = AddItem(itemType, *item); err != nil { + if err = h.AddItem(itemType, *item); err != nil { return fmt.Errorf("unable to add %s: %w", item.Name, err) } @@ -132,17 +131,17 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo disabled := 0 // remove all - for _, v := range GetItemMap(itemType) { + for _, v := range h.GetItemMap(itemType) { if !v.Installed { continue } - err := DisableItem(csConfig.Hub, &v, purge, forceAction) + err := h.DisableItem(&v, purge, forceAction) if err != nil { return fmt.Errorf("unable to disable %s: %w", v.Name, err) } - if err := AddItem(itemType, v); err != nil { + if err := h.AddItem(itemType, v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ @@ -154,11 +153,11 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo } // UpgradeConfig upgrades an item from the hub -func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error { +func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { updated := 0 found := false - for _, v := range GetItemMap(itemType) { + for _, v := range h.GetItemMap(itemType) { if name != "" && name != v.Name { continue } @@ -178,7 +177,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc if v.UpToDate { log.Infof("%s: up-to-date", v.Name) - if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { + if err := h.DownloadDataIfNeeded(v, force); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -187,7 +186,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } } - if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { + if err := h.DownloadLatest(&v, force, true); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -205,7 +204,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc updated++ } - if err := AddItem(itemType, v); err != nil { + if err := h.AddItem(itemType, v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index ecb778fdc54..72dac2342a2 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -4,165 +4,165 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err := InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) require.NoError(t, err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - if err = UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err = InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) - getHubIdxOrFail(t) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) } -func getHubIdxOrFail(t *testing.T) { - if err := GetHubIdx(getTestCfg().Hub); err != nil { - t.Fatalf("failed to load hub index") - } +// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test +func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { + hub, err := InitHub(hubCfg) + require.NoError(t, err, "failed to load hub index") + return hub } // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) require.NoError(t, err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - if err = UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err = InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - getHubIdxOrFail(t) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) - err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) - getHubIdxOrFail(t) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } func assertCollectionDepsInstalled(t *testing.T, collection string) { t.Helper() - c := hubIdx.Items[COLLECTIONS][collection] - require.NoError(t, CollecDepsCheck(&c)) + hub, err := GetHub() + require.NoError(t, err) + + c := hub.Items[COLLECTIONS][collection] + require.NoError(t, hub.CollectDepsCheck(&c)) } func pushUpdateToCollectionInHub() { diff --git a/pkg/cwhub/hubindex.go b/pkg/cwhub/hub.go similarity index 76% rename from pkg/cwhub/hubindex.go rename to pkg/cwhub/hub.go index a29cd873a25..6c9dce94b4d 100644 --- a/pkg/cwhub/hubindex.go +++ b/pkg/cwhub/hub.go @@ -6,8 +6,9 @@ import ( "strings" log "github.com/sirupsen/logrus" -) + "github.com/crowdsecurity/crowdsec/pkg/csconfig" +) const ( HubIndexFile = ".index.json" @@ -20,25 +21,34 @@ const ( ) var ( - // XXX: The order is important, as it is used to construct the - // index tree in memory --> collections must be last + // XXX: The order is important, as it is used to range over sub-items in collections ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} - hubIdx = HubIndex{} ) - type HubItems map[string]map[string]Item -// HubIndex represents the runtime status of the hub (parsed items, etc.) -// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared -type HubIndex struct { - Items HubItems +// Hub represents the runtime status of the hub (parsed items, etc.) +type Hub struct { + Items HubItems + cfg *csconfig.HubCfg skippedLocal int skippedTainted int } +var theHub *Hub + +// GetHub returns the hub singleton +// it returns an error if it's not initialized to avoid nil dereference +func GetHub() (*Hub, error) { + if theHub == nil { + return nil, fmt.Errorf("hub not initialized") + } + + return theHub, nil +} + // displaySummary prints a total count of the hub items -func (h HubIndex) displaySummary() { +func (h Hub) displaySummary() { msg := "Loaded: " for itemType := range h.Items { msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) @@ -52,8 +62,14 @@ func (h HubIndex) displaySummary() { // DisplaySummary prints a total count of the hub items. // It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton -func DisplaySummary() { - hubIdx.displaySummary() +// XXX: to be removed later +func DisplaySummary() error { + hub, err := GetHub() + if err != nil { + return err + } + hub.displaySummary() + return nil } // ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 4ac955b7b47..71fb46b1b97 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -6,12 +6,10 @@ import ( "path/filepath" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { - itempath := hub.HubDir + "/" + target.RemotePath +func (h *Hub) purgeItem(target Item) (Item, error) { + itempath := h.cfg.HubDir + "/" + target.RemotePath // disable hub file if err := os.Remove(itempath); err != nil { @@ -20,19 +18,19 @@ func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { target.Downloaded = false log.Infof("Removed source file [%s]: %s", target.Name, itempath) - hubIdx.Items[target.Type][target.Name] = target + h.Items[target.Type][target.Name] = target return target, nil } // DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error { +func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { var err error // already disabled, noop unless purge if !target.Installed { if purge { - *target, err = purgeItem(hub, *target) + *target, err = h.purgeItem(*target) if err != nil { return err } @@ -54,7 +52,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - if val, ok := hubIdx.Items[ptrtype][p]; ok { + if val, ok := h.Items[ptrtype][p]; ok { // check if the item doesn't belong to another collection before removing it toRemove := true @@ -66,7 +64,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err } if toRemove { - err = DisableItem(hub, &val, purge, force) + err = h.DisableItem(&val, purge, force) if err != nil { return fmt.Errorf("while disabling %s: %w", p, err) } @@ -80,7 +78,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err } } - syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) + syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) if err != nil { return err } @@ -103,7 +101,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err return fmt.Errorf("while reading symlink: %w", err) } - absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) + absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while abs path: %w", err) } @@ -124,23 +122,23 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err target.Installed = false if purge { - *target, err = purgeItem(hub, *target) + *target, err = h.purgeItem(*target) if err != nil { return err } } - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -func EnableItem(hub *csconfig.HubCfg, target *Item) error { +func (h *Hub) EnableItem(target *Item) error { var err error - parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/") + parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") // create directories if needed if target.Installed { @@ -172,12 +170,12 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx.Items[ptrtype][p] + val, ok := h.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } - err = EnableItem(hub, &val) + err = h.EnableItem(&val) if err != nil { return fmt.Errorf("while installing %s: %w", p, err) } @@ -192,7 +190,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { } // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) + srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while getting source path: %w", err) } @@ -208,7 +206,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { log.Infof("Enabled %s : %s", target.Type, target.Name) target.Installed = true - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 4cbceb4135c..31e347067d5 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -19,10 +19,6 @@ func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } -func validItemFileName(vname string, fauthor string, fname string) bool { - return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml") -} - func handleSymlink(path string) (string, error) { hubpath, err := os.Readlink(path) if err != nil { @@ -60,19 +56,6 @@ func getSHA256(filepath string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -type Walker struct { - // the walk/parserVisit function can't receive extra args - hubdir string - installdir string -} - -func NewWalker(hub *csconfig.HubCfg) Walker { - return Walker{ - hubdir: hub.HubDir, - installdir: hub.InstallDir, - } -} - type itemFileInfo struct { fname string stage string @@ -80,16 +63,19 @@ type itemFileInfo struct { fauthor string } -func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { +func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { ret := itemFileInfo{} inhub := false + hubDir := h.cfg.HubDir + installDir := h.cfg.InstallDir + subs := strings.Split(path, string(os.PathSeparator)) - log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir) + log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir) log.Tracef("subs:%v", subs) // we're in hub (~/.hub/hub/) - if strings.HasPrefix(path, w.hubdir) { + if strings.HasPrefix(path, hubDir) { log.Tracef("in hub dir") inhub = true @@ -104,7 +90,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { ret.fauthor = subs[len(subs)-2] ret.stage = subs[len(subs)-3] ret.ftype = subs[len(subs)-4] - } else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec//... + } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) @@ -118,7 +104,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { ret.ftype = subs[len(subs)-3] ret.fauthor = "" } else { - return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir) + return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) } log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) @@ -140,7 +126,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { return ret, inhub, nil } -func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { +func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { var ( local bool hubpath string @@ -166,7 +152,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { return nil } - info, inhub, err := w.getItemInfo(path) + info, inhub, err := h.getItemInfo(path) if err != nil { return err } @@ -197,12 +183,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { // if it's not a symlink and not in hub, it's a local file, don't bother if local && !inhub { log.Tracef("%s is a local file, skip", path) - hubIdx.skippedLocal++ + h.skippedLocal++ // log.Infof("local scenario, skip.") _, fileName := filepath.Split(path) - hubIdx.Items[info.ftype][info.fname] = Item{ + h.Items[info.ftype][info.fname] = Item{ Name: info.fname, Stage: info.stage, Installed: true, @@ -221,7 +207,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { match := false - for name, item := range hubIdx.Items[info.ftype] { + for name, item := range h.Items[info.ftype] { log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") if info.fname != item.FileName { @@ -241,12 +227,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { continue } - // wrong file - if !validItemFileName(item.Name, info.fauthor, info.fname) { + // not the item we're looking for + if !item.validPath(info.fauthor, info.fname) { continue } - if path == w.hubdir+"/"+item.RemotePath { + if path == h.cfg.HubDir+"/"+item.RemotePath { log.Tracef("marking %s as downloaded", item.Name) item.Downloaded = true } @@ -303,7 +289,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { if !match { log.Tracef("got tainted match for %s: %s", item.Name, path) - hubIdx.skippedTainted++ + h.skippedTainted++ // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { item.LocalPath = path @@ -316,7 +302,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { item.LocalHash = sha } - hubIdx.Items[info.ftype][name] = item + h.Items[info.ftype][name] = item return nil } @@ -326,13 +312,13 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { return nil } -func CollecDepsCheck(v *Item) error { - if v.versionStatus() != 0 { // not up-to-date - log.Debugf("%s dependencies not checked : not up-to-date", v.Name) +func (h *Hub) CollectDepsCheck(v *Item) error { + if v.Type != COLLECTIONS { return nil } - if v.Type != COLLECTIONS { + if v.versionStatus() != 0 { // not up-to-date + log.Debugf("%s dependencies not checked: not up-to-date", v.Name) return nil } @@ -342,7 +328,7 @@ func CollecDepsCheck(v *Item) error { for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { sliceType := ItemTypes[idx] for _, subName := range itemSlice { - subItem, ok := hubIdx.Items[sliceType][subName] + subItem, ok := h.Items[sliceType][subName] if !ok { return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) } @@ -356,7 +342,7 @@ func CollecDepsCheck(v *Item) error { if subItem.Type == COLLECTIONS { log.Tracef("collec, recurse.") - if err := CollecDepsCheck(&subItem); err != nil { + if err := h.CollectDepsCheck(&subItem); err != nil { if subItem.Tainted { v.Tainted = true } @@ -364,7 +350,7 @@ func CollecDepsCheck(v *Item) error { return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - hubIdx.Items[sliceType][subName] = subItem + h.Items[sliceType][subName] = subItem } // propagate the state of sub-items to set @@ -395,7 +381,7 @@ func CollecDepsCheck(v *Item) error { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } - hubIdx.Items[sliceType][subName] = subItem + h.Items[sliceType][subName] = subItem log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) } @@ -404,7 +390,7 @@ func CollecDepsCheck(v *Item) error { return nil } -func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { +func (h *Hub) SyncDir(dir string) ([]string, error) { warnings := []string{} // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last @@ -414,13 +400,13 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { log.Errorf("failed %s : %s", cpath, err) } - err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit) + err = filepath.WalkDir(cpath, h.itemVisit) if err != nil { return warnings, err } } - for name, item := range hubIdx.Items[COLLECTIONS] { + for name, item := range h.Items[COLLECTIONS] { if !item.Installed { continue } @@ -428,9 +414,9 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { vs := item.versionStatus() switch vs { case 0: // latest - if err := CollecDepsCheck(&item); err != nil { + if err := h.CollectDepsCheck(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - hubIdx.Items[COLLECTIONS][name] = item + h.Items[COLLECTIONS][name] = item } case 1: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) @@ -445,51 +431,55 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { } // Updates the info from HubInit() with the local state -func LocalSync(hub *csconfig.HubCfg) ([]string, error) { - hubIdx.skippedLocal = 0 - hubIdx.skippedTainted = 0 +func (h *Hub) LocalSync() ([]string, error) { + h.skippedLocal = 0 + h.skippedTainted = 0 - warnings, err := SyncDir(hub, hub.InstallDir) + warnings, err := h.SyncDir(h.cfg.InstallDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err) } - _, err = SyncDir(hub, hub.HubDir) + _, err = h.SyncDir(h.cfg.HubDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err) } return warnings, nil } -func GetHubIdx(hub *csconfig.HubCfg) error { - if hub == nil { - return fmt.Errorf("no configuration found for hub") +// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use +func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") } - log.Debugf("loading hub idx %s", hub.HubIndexFile) + log.Debugf("loading hub idx %s", cfg.HubIndexFile) - bidx, err := os.ReadFile(hub.HubIndexFile) + bidx, err := os.ReadFile(cfg.HubIndexFile) if err != nil { - return fmt.Errorf("unable to read index file: %w", err) + return nil, fmt.Errorf("unable to read index file: %w", err) } ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("unable to load existing index: %w", err) + return nil, fmt.Errorf("unable to load existing index: %w", err) } // XXX: why the error check if we bail out anyway? - return err + return nil, err } - hubIdx = HubIndex{Items: ret} + theHub = &Hub{ + Items: ret, + cfg: cfg, + } - _, err = LocalSync(hub) + _, err = theHub.LocalSync() if err != nil { - return fmt.Errorf("failed to sync Hub index with local deployment : %w", err) + return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) } - return nil + return theHub, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index eff2aa8fb0e..f8d9dce5f12 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -18,7 +18,7 @@ type HubTest struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *cwhub.HubIndex + HubIndex *cwhub.Hub Tests []*HubTestItem } @@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &cwhub.HubIndex{Items: hubIndex}, + HubIndex: &cwhub.Hub{Items: hubIndex}, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 25a89d8801d..f2b309bb54c 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -52,7 +52,7 @@ type HubTestItem struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *cwhub.HubIndex + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -391,16 +391,16 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - err := cwhub.GetHubIdx(t.RuntimeHubConfig) + hub, err := cwhub.InitHub(t.RuntimeHubConfig) if err != nil { - log.Fatalf("can't local sync the hub: %+v", err) + log.Fatal(err) } // install data for parsers if needed - ret := cwhub.GetItemMap(cwhub.PARSERS) + ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } log.Debugf("parser '%s' installed successfully in runtime environment", parserName) @@ -408,10 +408,10 @@ func (t *HubTestItem) InstallHub() error { } // install data for scenarios if needed - ret = cwhub.GetItemMap(cwhub.SCENARIOS) + ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) @@ -419,10 +419,10 @@ func (t *HubTestItem) InstallHub() error { } // install data for postoverflows if needed - ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS) + ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index e08887be818..41d463c489a 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -8,12 +8,14 @@ import ( "html/template" "io" "os" + "path/filepath" "reflect" "sync" "testing" "time" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -33,7 +35,20 @@ func TestBucket(t *testing.T) { envSetting = os.Getenv("TEST_ONLY") tomb = &tomb.Tomb{} ) - err := exprhelpers.Init(nil) + + testdata := "./tests" + + hubCfg := &csconfig.HubCfg{ + HubDir: filepath.Join(testdata, "hub"), + HubIndexFile: filepath.Join(testdata, "hub", "index.json"), + } + + _, err := cwhub.InitHub(hubCfg) + if err != nil { + t.Fatalf("failed to init hub : %s", err) + } + + err = exprhelpers.Init(nil) if err != nil { log.Fatalf("exprhelpers init failed: %s", err) } @@ -44,12 +59,15 @@ func TestBucket(t *testing.T) { } } else { wg := new(sync.WaitGroup) - fds, err := os.ReadDir("./tests/") + fds, err := os.ReadDir(testdata) if err != nil { t.Fatalf("Unable to read test directory : %s", err) } for _, fd := range fds { - fname := "./tests/" + fd.Name() + if fd.Name() == "hub" { + continue + } + fname := filepath.Join(testdata, fd.Name()) log.Infof("Running test on %s", fname) tomb.Go(func() error { wg.Add(1) @@ -112,10 +130,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { files = append(files, x.Filename) } - cscfg := &csconfig.CrowdsecServiceCfg{ - DataDir: "tests", - } - holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false) + cscfg := &csconfig.CrowdsecServiceCfg{} + holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false) if err != nil { t.Fatalf("failed loading bucket : %s", err) } @@ -123,7 +139,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { watchTomb(tomb) return nil }) - if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) { + if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) { return fmt.Errorf("tests from %s failed", dir) } return nil diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index b384e596712..484d6b9ac81 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -179,12 +179,17 @@ func ValidateFactory(bucketFactory *BucketFactory) error { return nil } -func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { +func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { var ( ret = []BucketFactory{} response chan types.Event ) + hub, err := cwhub.GetHub() + if err != nil { + return nil, nil, err + } + response = make(chan types.Event, 1) for _, f := range files { log.Debugf("Loading '%s'", f) @@ -212,7 +217,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. log.Tracef("End of yaml file") break } - bucketFactory.DataDir = cscfg.DataDir + bucketFactory.DataDir = dataDir //check empty if bucketFactory.Name == "" { log.Errorf("Won't load nameless bucket") @@ -235,7 +240,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. bucketFactory.Filename = filepath.Clean(f) bucketFactory.BucketName = seed.Generate() bucketFactory.ret = response - hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) + hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) if err != nil { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } else { diff --git a/pkg/leakybucket/tests/hub/index.json b/pkg/leakybucket/tests/hub/index.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/pkg/leakybucket/tests/hub/index.json @@ -0,0 +1 @@ +{} diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 48b09795bdf..4c2ae9fde57 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -64,8 +64,12 @@ func NewParsers() *Parsers { StageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0), } + + // XXX: handle error + hub, _ := cwhub.GetHub() + for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { - for _, hubParserItem := range cwhub.GetItemMap(itemType) { + for _, hubParserItem := range hub.GetItemMap(itemType) { if hubParserItem.Installed { stagefile := Stagefile{ Filename: hubParserItem.LocalPath, @@ -97,16 +101,16 @@ func NewParsers() *Parsers { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error - patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/") + patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/") log.Infof("Loading grok library %s", patternsDir) /* load base regexps for two grok parsers */ parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load parser patterns : %v", err) } parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err) } @@ -116,7 +120,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { */ log.Infof("Loading enrich plugins") - parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) + parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir) if err != nil { return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 4b603400955..347340253d4 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -54,7 +54,8 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error cwhub.SetHubBranch() - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + hub, err := cwhub.InitHub(csConfig.Hub) + if err != nil { return fmt.Errorf("getting hub index: %w", err) } @@ -75,7 +76,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing collection %s: %w", collection, err) } } @@ -89,7 +90,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing parser %s: %w", parser, err) } } @@ -103,7 +104,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing scenario %s: %w", scenario, err) } } @@ -117,7 +118,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } } From ef118a49ffe6a14b0fea854d3cc0f99e4fe7c691 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 19 Oct 2023 16:53:00 +0200 Subject: [PATCH 105/263] add waap-configs hub item --- cmd/crowdsec-cli/main.go | 1 + cmd/crowdsec-cli/waap_configs.go | 320 +++++++++++++++++++++++++++++++ pkg/cwhub/cwhub.go | 1 + pkg/cwhub/hub.go | 5 +- pkg/cwhub/install.go | 2 +- 5 files changed, 326 insertions(+), 3 deletions(-) create mode 100644 cmd/crowdsec-cli/waap_configs.go diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 59f737d30a3..7c0a3f40140 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -247,6 +247,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) rootCmd.AddCommand(NewWaapRulesCmd()) + rootCmd.AddCommand(NewWaapConfigsCmd()) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/waap_configs.go b/cmd/crowdsec-cli/waap_configs.go new file mode 100644 index 00000000000..3755bc8180c --- /dev/null +++ b/cmd/crowdsec-cli/waap_configs.go @@ -0,0 +1,320 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewWaapConfigsCmd() *cobra.Command { + cmdWaapConfigs := &cobra.Command{ + Use: "waap-configs [waap-config]...", + Short: "Manage hub waap configs", + Example: `cscli waap-configs list -a +cscli waap-configs install crowdsecurity/crs +cscli waap-configs inspect crowdsecurity/crs +cscli waap-configs upgrade crowdsecurity/crs +cscli waap-configs remove crowdsecurity/crs +`, + Args: cobra.MinimumNArgs(1), + Aliases: []string{"waap-config"}, + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if _, err := require.Hub(csConfig); err != nil { + return err + } + + return nil + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + if cmd.Name() == "inspect" || cmd.Name() == "list" { + return + } + log.Infof(ReloadMessage()) + }, + } + + cmdWaapConfigs.AddCommand(NewCmdWaapConfigsInstall()) + cmdWaapConfigs.AddCommand(NewCmdWaapConfigsRemove()) + cmdWaapConfigs.AddCommand(NewCmdWaapConfigsUpgrade()) + cmdWaapConfigs.AddCommand(NewCmdWaapConfigsInspect()) + cmdWaapConfigs.AddCommand(NewCmdWaapConfigsList()) + + return cmdWaapConfigs +} + +func runWaapConfigsInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + for _, name := range args { + t := hub.GetItem(cwhub.WAAP_CONFIGS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.WAAP_CONFIGS, name) + Suggest(cwhub.WAAP_CONFIGS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := hub.InstallItem(name, cwhub.WAAP_CONFIGS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewCmdWaapConfigsInstall() *cobra.Command { + cmdWaapConfigsInstall := &cobra.Command{ + Use: "install ...", + Short: "Install given waap config(s)", + Long: `Fetch and install one or more waap configs from the hub`, + Example: `cscli waap-configs install crowdsecurity/vpatch`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(cwhub.WAAP_CONFIGS, args, toComplete) + }, + RunE: runWaapConfigsInstall, + } + + flags := cmdWaapConfigsInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple waap rules") + + return cmdWaapConfigsInstall +} + +func runWaapConfigsRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + err := hub.RemoveMany(cwhub.WAAP_CONFIGS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one waap rule to remove or '--all'") + } + + for _, name := range args { + err := hub.RemoveMany(cwhub.WAAP_CONFIGS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapConfigsRemove() *cobra.Command { + cmdWaapConfigsRemove := &cobra.Command{ + Use: "remove ...", + Short: "Remove given waap config(s)", + Long: `remove one or more waap configs`, + Example: `cscli waap-configs remove crowdsecurity/vpatch`, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) + }, + RunE: runWaapConfigsRemove, + } + + flags := cmdWaapConfigsRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the waap configs") + + return cmdWaapConfigsRemove +} + +func runWaapConfigsUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + if err := hub.UpgradeConfig(cwhub.WAAP_CONFIGS, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one waap config to upgrade or '--all'") + } + + for _, name := range args { + if err := hub.UpgradeConfig(cwhub.WAAP_CONFIGS, name, force); err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapConfigsUpgrade() *cobra.Command { + cmdWaapConfigsUpgrade := &cobra.Command{ + Use: "upgrade ...", + Short: "Upgrade given waap config(s)", + Long: `Fetch and upgrade one or more waap configs from the hub`, + Example: `cscli waap-configs upgrade crowdsecurity/crs`, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) + }, + RunE: runWaapConfigsUpgrade, + } + + flags := cmdWaapConfigsUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the waap configs") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmdWaapConfigsUpgrade +} + +func runWaapConfigsInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.WAAP_CONFIGS, noMetrics); err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapConfigsInspect() *cobra.Command { + cmdWaapConfigsInspect := &cobra.Command{ + Use: "inspect ", + Short: "Inspect a waap config", + Long: `Inspect a waap config`, + Example: `cscli waap-configs inspect crowdsecurity/vpatch`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) + }, + RunE: runWaapConfigsInspect, + } + + flags := cmdWaapConfigsInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmdWaapConfigsInspect +} + +func runWaapConfigsList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = ListItems(color.Output, []string{cwhub.WAAP_CONFIGS}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewCmdWaapConfigsList() *cobra.Command { + cmdWaapConfigsList := &cobra.Command{ + Use: "list [waap-config]...", + Short: "List waap configs", + Long: `List of installed/available/specified waap configs`, + Example: `cscli waap-configs list +cscli waap-configs list -a +cscli waap-configs list crowdsecurity/crs`, + DisableAutoGenTag: true, + RunE: runWaapConfigsList, + } + + flags := cmdWaapConfigsList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmdWaapConfigsList +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index d55b2fad464..4bcc25fe707 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -63,6 +63,7 @@ type Item struct { Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` WaapRules []string `json:"waap-rules,omitempty" yaml:"waap-rules,omitempty"` + WaapConfigs []string `json:"waap-configs,omitempty" yaml:"waap-configs,omitempty"` } // Status returns the status of the item as a string and an emoji diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index ba94c782167..fc7f263bfa8 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -19,11 +19,12 @@ const ( POSTOVERFLOWS = "postoverflows" SCENARIOS = "scenarios" WAAP_RULES = "waap-rules" + WAAP_CONFIGS = "waap-configs" ) var ( // XXX: The order is important, as it is used to range over sub-items in collections - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, WAAP_RULES, COLLECTIONS} + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, WAAP_RULES, WAAP_CONFIGS, COLLECTIONS} ) type HubItems map[string]map[string]Item @@ -107,7 +108,7 @@ func ParseIndex(buff []byte) (HubItems, error) { // if it's a collection, check its sub-items are present // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.WaapRules, item.Collections} { + for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.WaapRules, item.WaapConfigs, item.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { if _, ok := RawIndex[ptrtype][p]; !ok { diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 71fb46b1b97..0a71088effd 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -49,7 +49,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { // for a COLLECTIONS, disable sub-items if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { + for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.WaapRules, target.WaapConfigs, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { if val, ok := h.Items[ptrtype][p]; ok { From 68c78249d5eba12354e954712b0260cc8109a2cd Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 19 Oct 2023 17:20:33 +0200 Subject: [PATCH 106/263] up --- pkg/cwhub/loader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 2bd435d1c18..7f59e9e035d 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -119,6 +119,9 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { } else if ret.stage == WAAP_RULES { ret.ftype = WAAP_RULES ret.stage = "" + } else if ret.stage == WAAP_CONFIGS { + ret.ftype = WAAP_CONFIGS + ret.stage = "" } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { // its a PARSER / PARSER_OVFLW with a stage return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) From 1468bb96814854c7939d7d916b0fc6d330afa8c6 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 19 Oct 2023 17:25:48 +0200 Subject: [PATCH 107/263] up --- pkg/cwhub/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 7f59e9e035d..7a2ee52217f 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -331,7 +331,7 @@ func (h *Hub) CollectDepsCheck(v *Item) error { // if it's a collection, ensure all the items are installed, or tag it as tainted log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) - for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { + for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.WaapRules, v.WaapConfigs, v.Collections} { sliceType := ItemTypes[idx] for _, subName := range itemSlice { subItem, ok := h.Items[sliceType][subName] From 0acda36d33b5cb9e213509dd768e2c20048f66f7 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 20 Oct 2023 11:58:57 +0200 Subject: [PATCH 108/263] up --- pkg/cwhub/download.go | 2 +- pkg/cwhub/install.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 0f4a8fb7a7a..4ff09b91468 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -121,7 +121,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro } // collection - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} + var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.WaapRules, target.WaapConfigs, target.Collections} for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 0a71088effd..2677a013b6c 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -167,7 +167,7 @@ func (h *Hub) EnableItem(target *Item) error { // install sub-items if it's a collection if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { + for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.WaapRules, target.WaapConfigs, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { val, ok := h.Items[ptrtype][p] From 5dbc2758fa955b7790cc6cafa24831a534a9671d Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 20 Oct 2023 13:32:20 +0200 Subject: [PATCH 109/263] warn user when setting unexpected default_remediation --- pkg/waf/waap.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 0bafff1a80d..21f7229325c 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -84,9 +84,9 @@ type WaapConfig struct { } func (w *WaapRuntimeConfig) ClearResponse() { - log.Infof("#-> %p", w) + log.Debugf("#-> %p", w) w.Response = WaapTempResponse{} - log.Infof("-> %p", w.Config) + log.Debugf("-> %p", w.Config) w.Response.Action = w.Config.DefaultPassAction w.Response.HTTPResponseCode = w.Config.PassedHTTPCode w.Response.SendEvent = true @@ -110,6 +110,12 @@ func (wc *WaapConfig) Load(file string) error { if wc.DefaultRemediation == "" { return fmt.Errorf("default_remediation cannot be empty") } + switch wc.DefaultRemediation { + case "ban", "captcha", "log": + //those are the officially supported remediation(s) + default: + wc.Logger.Warningf("default '%s' remediation of %s is none of [ban,captcha,log] ensure bouncer compatbility!", wc.DefaultRemediation, file) + } if wc.BlockedHTTPCode == 0 { wc.BlockedHTTPCode = 403 } From b110c74487ace275a162d4ad36953e5e38030134 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 20 Oct 2023 13:49:15 +0200 Subject: [PATCH 110/263] allow description --- pkg/waf/waap_rules_collection.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 2a590427fef..14e25e74c9a 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -25,6 +25,7 @@ var WAAP_RULE = "waap-rule" type WaapCollectionConfig struct { Type string `yaml:"type"` Name string `yaml:"name"` + Description string `yaml:"description"` SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` Rules []VPatchRule `yaml:"rules"` From ac98256602bb6970ea224a1f7dc19925b51a0558 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:32:35 +0200 Subject: [PATCH 111/263] Refact pkg/cwhub, cmd/crowdsec-cli (#2557) - pkg/cwhub: change file layout, rename functions - method Item.SubItems - cmd/crowdsec-cli: generic code for hub items - cscli: removing any type of items in a collection now requires --force - tests --- cmd/crowdsec-cli/collections.go | 333 ----------------- cmd/crowdsec-cli/config_restore.go | 4 +- cmd/crowdsec-cli/hub.go | 4 +- cmd/crowdsec-cli/itemcommands.go | 503 ++++++++++++++++++++++++++ cmd/crowdsec-cli/main.go | 8 +- cmd/crowdsec-cli/parsers.go | 320 ---------------- cmd/crowdsec-cli/postoverflows.go | 321 ---------------- cmd/crowdsec-cli/scenarios.go | 320 ---------------- go.mod | 2 +- go.sum | 4 +- pkg/cwhub/branch.go | 59 +++ pkg/cwhub/cwhub.go | 236 +----------- pkg/cwhub/cwhub_test.go | 231 +----------- pkg/cwhub/dataset.go | 41 ++- pkg/cwhub/dataset_test.go | 6 + pkg/cwhub/download.go | 336 ----------------- pkg/cwhub/download_test.go | 60 --- pkg/cwhub/{install.go => enable.go} | 203 ++++++----- pkg/cwhub/enable_test.go | 144 ++++++++ pkg/cwhub/helpers.go | 261 ++++++++++--- pkg/cwhub/helpers_test.go | 7 +- pkg/cwhub/hub.go | 200 +++++++--- pkg/cwhub/hub_test.go | 63 ++++ pkg/cwhub/items.go | 232 ++++++++++++ pkg/cwhub/items_test.go | 75 ++++ pkg/cwhub/leakybucket.go | 58 +++ pkg/cwhub/{loader.go => sync.go} | 126 +++---- pkg/hubtest/utils.go | 2 +- test/bats/20_hub_collections.bats | 8 + test/bats/20_hub_collections_dep.bats | 4 +- test/bats/20_hub_parsers.bats | 8 + test/bats/20_hub_postoverflows.bats | 8 + test/bats/20_hub_scenarios.bats | 8 + 33 files changed, 1741 insertions(+), 2454 deletions(-) delete mode 100644 cmd/crowdsec-cli/collections.go create mode 100644 cmd/crowdsec-cli/itemcommands.go delete mode 100644 cmd/crowdsec-cli/parsers.go delete mode 100644 cmd/crowdsec-cli/postoverflows.go delete mode 100644 cmd/crowdsec-cli/scenarios.go create mode 100644 pkg/cwhub/branch.go delete mode 100644 pkg/cwhub/download.go delete mode 100644 pkg/cwhub/download_test.go rename pkg/cwhub/{install.go => enable.go} (78%) create mode 100644 pkg/cwhub/enable_test.go create mode 100644 pkg/cwhub/hub_test.go create mode 100644 pkg/cwhub/items.go create mode 100644 pkg/cwhub/items_test.go create mode 100644 pkg/cwhub/leakybucket.go rename pkg/cwhub/{loader.go => sync.go} (80%) diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go deleted file mode 100644 index 494cedab27e..00000000000 --- a/cmd/crowdsec-cli/collections.go +++ /dev/null @@ -1,333 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewCollectionsCmd() *cobra.Command { - cmdCollections := &cobra.Command{ - Use: "collections [collection]...", - Short: "Manage hub collections", - Example: `cscli collections list -a -cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables -cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"collection"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdCollections.AddCommand(NewCollectionsInstallCmd()) - cmdCollections.AddCommand(NewCollectionsRemoveCmd()) - cmdCollections.AddCommand(NewCollectionsUpgradeCmd()) - cmdCollections.AddCommand(NewCollectionsInspectCmd()) - cmdCollections.AddCommand(NewCollectionsListCmd()) - - return cmdCollections -} - -func runCollectionsInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.COLLECTIONS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) - Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewCollectionsInstallCmd() *cobra.Command { - cmdCollectionsInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given collection(s)", - Long: `Fetch and install one or more collections from hub`, - Example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsInstall, - } - - flags := cmdCollectionsInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple collections") - - return cmdCollectionsInstall -} - -func runCollectionsRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to remove or '--all'") - } - - for _, name := range args { - if !force { - item := hub.GetItem(cwhub.COLLECTIONS, name) - if item == nil { - // XXX: this should be in GetItem? - return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections) - log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name) - continue - } - } - - err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewCollectionsRemoveCmd() *cobra.Command { - cmdCollectionsRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given collection(s)", - Long: `Remove one or more collections`, - Example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsRemove, - } - - flags := cmdCollectionsRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the collections") - - return cmdCollectionsRemove -} - -func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewCollectionsUpgradeCmd() *cobra.Command { - cmdCollectionsUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given collection(s)", - Long: `Fetch and upgrade one or more collections from the hub`, - Example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsUpgrade, - } - - flags := cmdCollectionsUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the collections") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdCollectionsUpgrade -} - -func runCollectionsInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewCollectionsInspectCmd() *cobra.Command { - cmdCollectionsInspect := &cobra.Command{ - Use: "inspect ...", - Short: "Inspect given collection(s)", - Long: `Inspect one or more collections`, - Example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) - }, - RunE: runCollectionsInspect, - } - - flags := cmdCollectionsInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdCollectionsInspect -} - -func runCollectionsList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewCollectionsListCmd() *cobra.Command { - cmdCollectionsList := &cobra.Command{ - Use: "list [collection... | -a]", - Short: "List collections", - Long: `List of installed/available/specified collections`, - Example: `cscli collections list -cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, - DisableAutoGenTag: true, - RunE: runCollectionsList, - } - - flags := cmdCollectionsList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdCollectionsList -} diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index ba734461eb1..1a081d6c5ce 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -36,7 +36,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } - if err := hub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(*item); err != nil { return "", err } @@ -44,7 +44,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) } - if err := hub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(*item); err != nil { return "", err } return fmt.Sprintf("Enabled %s", item.Name), nil diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c4ec140e94e..c0f69ea870f 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -61,7 +61,9 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(v) } - cwhub.DisplaySummary() + for line := range hub.ItemStats() { + log.Info(line) + } err = ListItems(color.Output, []string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go new file mode 100644 index 00000000000..0afa27787eb --- /dev/null +++ b/cmd/crowdsec-cli/itemcommands.go @@ -0,0 +1,503 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/coalesce" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +type cmdHelp struct { + // Example is required, the others have a default value + // generated from the item type + use string + short string + long string + example string +} + +type hubItemType struct { + name string // plural, as used in the hub index + singular string + oneOrMore string // parenthetical pluralizaion: "parser(s)" + help cmdHelp + installHelp cmdHelp + removeHelp cmdHelp + upgradeHelp cmdHelp + inspectHelp cmdHelp + listHelp cmdHelp +} + +var hubItemTypes = map[string]hubItemType{ + "parsers": { + name: "parsers", + singular: "parser", + oneOrMore: "parser(s)", + help: cmdHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cmdHelp{ + example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + removeHelp: cmdHelp{ + example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + upgradeHelp: cmdHelp{ + example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + inspectHelp: cmdHelp{ + example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, + }, + listHelp: cmdHelp{ + example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + }, + "postoverflows": { + name: "postoverflows", + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cmdHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cmdHelp{ + example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + removeHelp: cmdHelp{ + example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + upgradeHelp: cmdHelp{ + example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + inspectHelp: cmdHelp{ + example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + listHelp: cmdHelp{ + example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + }, + "scenarios": { + name: "scenarios", + singular: "scenario", + oneOrMore: "scenario(s)", + help: cmdHelp{ + example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing +`, + }, + installHelp: cmdHelp{ + example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + removeHelp: cmdHelp{ + example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + upgradeHelp: cmdHelp{ + example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + inspectHelp: cmdHelp{ + example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + listHelp: cmdHelp{ + example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + }, + "collections": { + name: "collections", + singular: "collection", + oneOrMore: "collection(s)", + help: cmdHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cmdHelp{ + example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + }, + removeHelp: cmdHelp{ + example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, + }, + upgradeHelp: cmdHelp{ + example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, + }, + inspectHelp: cmdHelp{ + example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, + }, + listHelp: cmdHelp{ + example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, + }, + }, +} + +func NewItemsCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), + Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), + Long: it.help.long, + Example: it.help.example, + Args: cobra.MinimumNArgs(1), + Aliases: []string{it.singular}, + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if _, err := require.Hub(csConfig); err != nil { + return err + } + + return nil + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + if cmd.Name() == "inspect" || cmd.Name() == "list" { + return + } + log.Infof(ReloadMessage()) + }, + } + + cmd.AddCommand(NewItemsInstallCmd(typeName)) + cmd.AddCommand(NewItemsRemoveCmd(typeName)) + cmd.AddCommand(NewItemsUpgradeCmd(typeName)) + cmd.AddCommand(NewItemsInspectCmd(typeName)) + cmd.AddCommand(NewItemsListCmd(typeName)) + + return cmd +} + +func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + for _, name := range args { + t := hub.GetItem(it.name, name) + if t == nil { + nearestItem, score := GetDistance(it.name, name) + Suggest(it.name, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + return nil + } + + return run +} + +func NewItemsInstallCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.installHelp.use, "install [item]..."), + Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), + Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)), + Example: it.installHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(typeName, args, toComplete) + }, + RunE: itemsInstallRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name)) + + return cmd +} + +func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + err := hub.RemoveMany(it.name, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + } + + for _, name := range args { + if !force { + item := hub.GetItem(it.name, name) + if item == nil { + // XXX: this should be in GetItem? + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if len(item.BelongsToCollections) > 0 { + log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular) + continue + } + } + + err := hub.RemoveMany(it.name, name, all, purge, force) + if err != nil { + return err + } + } + + return nil + } + return run +} + +func NewItemsRemoveCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.removeHelp.use, "remove [item]..."), + Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), + Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)), + Example: it.removeHelp.example, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsRemoveRunner(it), + } + + flags := cmd.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name)) + + return cmd +} + +func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if all { + if err := hub.UpgradeConfig(it.name, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) + } + + for _, name := range args { + if err := hub.UpgradeConfig(it.name, name, force); err != nil { + return err + } + } + + return nil + } + + return run +} + +func NewItemsUpgradeCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), + Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), + Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)), + Example: it.upgradeHelp.example, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsUpgradeRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name)) + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmd +} + +func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, it.name, noMetrics); err != nil { + return err + } + } + + return nil + } + + return run +} + +func NewItemsInspectCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), + Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), + Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)), + Example: it.inspectHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: itemsInspectRunner(it), + } + + flags := cmd.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmd +} + +func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { + run := func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = ListItems(color.Output, []string{it.name}, args, false, true, all); err != nil { + return err + } + + return nil + } + + return run +} + +func NewItemsListCmd(typeName string) *cobra.Command { + it := hubItemTypes[typeName] + + cmd := &cobra.Command{ + Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), + Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), + Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), + Example: it.listHelp.example, + DisableAutoGenTag: true, + RunE: itemsListRunner(it), + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmd +} diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 1ddac4bf80c..1d3aa56a656 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -234,10 +234,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewSimulationCmds()) rootCmd.AddCommand(NewBouncersCmd()) rootCmd.AddCommand(NewMachinesCmd()) - rootCmd.AddCommand(NewParsersCmd()) - rootCmd.AddCommand(NewScenariosCmd()) - rootCmd.AddCommand(NewCollectionsCmd()) - rootCmd.AddCommand(NewPostOverflowsCmd()) rootCmd.AddCommand(NewCapiCmd()) rootCmd.AddCommand(NewLapiCmd()) rootCmd.AddCommand(NewCompletionCmd()) @@ -246,6 +242,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) + rootCmd.AddCommand(NewItemsCmd("collections")) + rootCmd.AddCommand(NewItemsCmd("parsers")) + rootCmd.AddCommand(NewItemsCmd("scenarios")) + rootCmd.AddCommand(NewItemsCmd("postoverflows")) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go deleted file mode 100644 index e0d294e0191..00000000000 --- a/cmd/crowdsec-cli/parsers.go +++ /dev/null @@ -1,320 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewParsersCmd() *cobra.Command { - cmdParsers := &cobra.Command{ - Use: "parsers [parser]...", - Short: "Manage hub parsers", - Example: `cscli parsers list -a -cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"parser"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdParsers.AddCommand(NewParsersInstallCmd()) - cmdParsers.AddCommand(NewParsersRemoveCmd()) - cmdParsers.AddCommand(NewParsersUpgradeCmd()) - cmdParsers.AddCommand(NewParsersInspectCmd()) - cmdParsers.AddCommand(NewParsersListCmd()) - - return cmdParsers -} - -func runParsersInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.PARSERS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS, name) - Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewParsersInstallCmd() *cobra.Command { - cmdParsersInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given parser(s)", - Long: `Fetch and install one or more parsers from the hub`, - Example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersInstall, - } - - flags := cmdParsersInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple parsers") - - return cmdParsersInstall -} - -func runParsersRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewParsersRemoveCmd() *cobra.Command { - cmdParsersRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given parser(s)", - Long: `Remove one or more parsers`, - Example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersRemove, - } - - flags := cmdParsersRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the parsers") - - return cmdParsersRemove -} - -func runParsersUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewParsersUpgradeCmd() *cobra.Command { - cmdParsersUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given parser(s)", - Long: `Fetch and upgrade one or more parsers from the hub`, - Example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersUpgrade, - } - - flags := cmdParsersUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the parsers") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdParsersUpgrade -} - -func runParsersInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewParsersInspectCmd() *cobra.Command { - cmdParsersInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a parser", - Long: `Inspect a parser`, - Example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS, args, toComplete) - }, - RunE: runParsersInspect, - } - - flags := cmdParsersInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdParsersInspect -} - -func runParsersList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewParsersListCmd() *cobra.Command { - cmdParsersList := &cobra.Command{ - Use: "list [parser... | -a]", - Short: "List parsers", - Long: `List of installed/available/specified parsers`, - Example: `cscli parsers list -cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - DisableAutoGenTag: true, - RunE: runParsersList, - } - - flags := cmdParsersList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdParsersList -} diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go deleted file mode 100644 index a73bf3da450..00000000000 --- a/cmd/crowdsec-cli/postoverflows.go +++ /dev/null @@ -1,321 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewPostOverflowsCmd() *cobra.Command { - cmdPostOverflows := &cobra.Command{ - Use: "postoverflows [postoverflow]...", - Short: "Manage hub postoverflows", - Example: `cscli postoverflows list -a -cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"postoverflow"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd()) - cmdPostOverflows.AddCommand(NewPostOverflowsListCmd()) - - return cmdPostOverflows -} - -func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.POSTOVERFLOWS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) - Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewPostOverflowsInstallCmd() *cobra.Command { - cmdPostOverflowsInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given postoverflow(s)", - Long: `Fetch and install one or more postoverflows from the hub`, - Example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsInstall, - } - - flags := cmdPostOverflowsInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows") - - return cmdPostOverflowsInstall -} - -func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsRemoveCmd() *cobra.Command { - cmdPostOverflowsRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given postoverflow(s)", - Long: `remove one or more postoverflows from the hub`, - Example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsRemove, - } - - flags := cmdPostOverflowsRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Delete all the postoverflows") - - return cmdPostOverflowsRemove -} - -func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsUpgradeCmd() *cobra.Command { - cmdPostOverflowsUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given postoverflow(s)", - Long: `Fetch and upgrade one or more postoverflows from the hub`, - Example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowUpgrade, - } - - flags := cmdPostOverflowsUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the postoverflows") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdPostOverflowsUpgrade -} - -func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewPostOverflowsInspectCmd() *cobra.Command { - cmdPostOverflowsInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a postoverflow", - Long: `Inspect a postoverflow`, - Example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) - }, - RunE: runPostOverflowsInspect, - } - - flags := cmdPostOverflowsInspect.Flags() - - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdPostOverflowsInspect -} - -func runPostOverflowsList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewPostOverflowsListCmd() *cobra.Command { - cmdPostOverflowsList := &cobra.Command{ - Use: "list [postoverflow]...", - Short: "List postoverflows", - Long: `List of installed/available/specified postoverflows`, - Example: `cscli postoverflows list -cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - DisableAutoGenTag: true, - RunE: runPostOverflowsList, - } - - flags := cmdPostOverflowsList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdPostOverflowsList -} diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go deleted file mode 100644 index 23df3f5ff01..00000000000 --- a/cmd/crowdsec-cli/scenarios.go +++ /dev/null @@ -1,320 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewScenariosCmd() *cobra.Command { - cmdScenarios := &cobra.Command{ - Use: "scenarios [scenario]...", - Short: "Manage hub scenarios", - Example: `cscli scenarios list -a -cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"scenario"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdScenarios.AddCommand(NewCmdScenariosInstall()) - cmdScenarios.AddCommand(NewCmdScenariosRemove()) - cmdScenarios.AddCommand(NewCmdScenariosUpgrade()) - cmdScenarios.AddCommand(NewCmdScenariosInspect()) - cmdScenarios.AddCommand(NewCmdScenariosList()) - - return cmdScenarios -} - -func runScenariosInstall(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - for _, name := range args { - t := hub.GetItem(cwhub.SCENARIOS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.SCENARIOS, name) - Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) - - continue - } - - if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - - return nil -} - -func NewCmdScenariosInstall() *cobra.Command { - cmdScenariosInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given scenario(s)", - Long: `Fetch and install one or more scenarios from the hub`, - Example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosInstall, - } - - flags := cmdScenariosInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios") - - return cmdScenariosInstall -} - -func runScenariosRemove(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force) - if err != nil { - return err - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to remove or '--all'") - } - - for _, name := range args { - err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force) - if err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosRemove() *cobra.Command { - cmdScenariosRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given scenario(s)", - Long: `remove one or more scenarios`, - Example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosRemove, - } - - flags := cmdScenariosRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the scenarios") - - return cmdScenariosRemove -} - -func runScenariosUpgrade(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if all { - if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { - return err - } - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to upgrade or '--all'") - } - - for _, name := range args { - if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosUpgrade() *cobra.Command { - cmdScenariosUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given scenario(s)", - Long: `Fetch and upgrade one or more scenarios from the hub`, - Example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosUpgrade, - } - - flags := cmdScenariosUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the scenarios") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdScenariosUpgrade -} - -func runScenariosInspect(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - for _, name := range args { - if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil { - return err - } - } - - return nil -} - -func NewCmdScenariosInspect() *cobra.Command { - cmdScenariosInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a scenario", - Long: `Inspect a scenario`, - Example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.SCENARIOS, args, toComplete) - }, - RunE: runScenariosInspect, - } - - flags := cmdScenariosInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdScenariosInspect -} - -func runScenariosList(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil { - return err - } - - return nil -} - -func NewCmdScenariosList() *cobra.Command { - cmdScenariosList := &cobra.Command{ - Use: "list [scenario]...", - Short: "List scenarios", - Long: `List of installed/available/specified scenarios`, - Example: `cscli scenarios list -cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, - DisableAutoGenTag: true, - RunE: runScenariosList, - } - - flags := cmdScenariosList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdScenariosList -} diff --git a/go.mod b/go.mod index 5d641b084e2..15b3ed07a2a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/c-robinson/iplib v1.0.3 github.com/cespare/xxhash/v2 v2.2.0 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.4 + github.com/crowdsecurity/go-cs-lib v0.0.5 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 5c9af48bd8d..82d02546a8f 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims= -github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= +github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/cwhub/branch.go b/pkg/cwhub/branch.go new file mode 100644 index 00000000000..e25a6becf31 --- /dev/null +++ b/pkg/cwhub/branch.go @@ -0,0 +1,59 @@ +package cwhub + +// Set the appropriate hub branch according to config settings and crowdsec version + +import ( + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/cwversion" +) + +// chooseHubBranch returns the branch name to use for the hub +// It can be "master" or the branch corresponding to the current crowdsec version +func chooseHubBranch() string { + latest, err := cwversion.Latest() + if err != nil { + log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) + return "master" + } + + csVersion := cwversion.VersionStrip() + if csVersion == latest { + log.Debugf("current version is equal to latest (%s)", csVersion) + return "master" + } + + // if current version is greater than the latest we are in pre-release + if semver.Compare(csVersion, latest) == 1 { + log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) + return "master" + } + + if csVersion == "" { + log.Warning("Crowdsec version is not set, using master branch for the hub") + return "master" + } + + log.Warnf("Crowdsec is not the latest version. "+ + "Current version is '%s' and the latest stable version is '%s'. Please update it!", + csVersion, latest) + + log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ + "added to Crowdsec Hub after CrowdSec %s", latest) + + return csVersion +} + +// SetHubBranch sets the package variable that points to the hub branch. +func SetHubBranch() { + // a branch is already set, or specified from the flags + if HubBranch != "" { + return + } + + // use the branch corresponding to the crowdsec version + HubBranch = chooseHubBranch() + + log.Debugf("Using branch '%s' for the hub", HubBranch) +} diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 7ae80549231..d1f155813a8 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -5,14 +5,7 @@ package cwhub import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/enescakir/emoji" - "github.com/pkg/errors" - "golang.org/x/mod/semver" + "errors" ) var ( @@ -21,230 +14,3 @@ var ( RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" HubBranch = "master" ) - -// ItemVersion is used to detect the version of a given item -// by comparing the hash of each version to the local file. -// If the item does not match any known version, it is considered tainted. -type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? -} - -// Item represents an object managed in the hub. It can be a parser, scenario, collection.. -type Item struct { - // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json - Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any - - // remote (hub) info - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml - Version string `json:"version,omitempty"` // the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - - // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} - LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed,omitempty"` - Downloaded bool `json:"downloaded,omitempty"` - UpToDate bool `json:"up_to_date,omitempty"` - Tainted bool `json:"tainted,omitempty"` // has it been locally modified - Local bool `json:"local,omitempty"` // if it's a non versioned control one - - // if it's a collection, it can have sub items - Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` - PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` - Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` - Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` -} - -// Status returns the status of the item as a string and an emoji -// ie. "enabled,update-available" and emoji.Warning -func (i *Item) Status() (string, emoji.Emoji) { - status := "disabled" - ok := false - - if i.Installed { - ok = true - status = "enabled" - } - - managed := true - if i.Local { - managed = false - status += ",local" - } - - warning := false - if i.Tainted { - warning = true - status += ",tainted" - } else if !i.UpToDate && !i.Local { - warning = true - status += ",update-available" - } - - emo := emoji.QuestionMark - - switch { - case !managed: - emo = emoji.House - case !i.Installed: - emo = emoji.Prohibited - case warning: - emo = emoji.Warning - case ok: - emo = emoji.CheckMark - } - - return status, emo -} - -// versionStatus: semver requires 'v' prefix -func (i *Item) versionStatus() int { - return semver.Compare("v"+i.Version, "v"+i.LocalVersion) -} - -// validPath returns true if the (relative) path is allowed for the item -// dirNmae: the directory name (ie. crowdsecurity) -// fileName: the filename (ie. apache2-logs.yaml) -func (i *Item) validPath(dirName, fileName string) bool { - return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") -} - -// GetItemMap returns the map of items for a given type -func (h *Hub) GetItemMap(itemType string) map[string]Item { - m, ok := h.Items[itemType] - if !ok { - return nil - } - - return m -} - -// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary -// XXX: only used by leakybucket manager -func itemKey(itemPath string) (string, error) { - f, err := os.Lstat(itemPath) - if err != nil { - return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) - } - - if f.Mode()&os.ModeSymlink == 0 { - // it's not a symlink, so the filename itsef should be the key - return filepath.Base(itemPath), nil - } - - // resolve the symlink to hub file - pathInHub, err := os.Readlink(itemPath) - if err != nil { - return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) - } - - author := filepath.Base(filepath.Dir(pathInHub)) - - fname := filepath.Base(pathInHub) - fname = strings.TrimSuffix(fname, ".yaml") - fname = strings.TrimSuffix(fname, ".yml") - - return fmt.Sprintf("%s/%s", author, fname), nil -} - -// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. -func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { - itemKey, err := itemKey(itemPath) - if err != nil { - return nil, err - } - - m := h.GetItemMap(itemType) - if m == nil { - return nil, fmt.Errorf("item type %s doesn't exist", itemType) - } - - v, ok := m[itemKey] - if !ok { - return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) - } - - return &v, nil -} - -// GetItem returns the item from hub based on its type and full name (author/name) -func (h *Hub) GetItem(itemType string, itemName string) *Item { - m, ok := h.GetItemMap(itemType)[itemName] - if !ok { - return nil - } - - return &m -} - -// GetItemNames returns the list of item (full) names for a given type -// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx -// The names can be used to retrieve the item with GetItem() -func (h *Hub) GetItemNames(itemType string) []string { - m := h.GetItemMap(itemType) - if m == nil { - return nil - } - - names := make([]string, 0, len(m)) - for k := range m { - names = append(names, k) - } - - return names -} - -// AddItem adds an item to the hub index -func (h *Hub) AddItem(itemType string, item Item) error { - for _, itype := range ItemTypes { - if itype == itemType { - h.Items[itemType][item.Name] = item - return nil - } - } - - return fmt.Errorf("ItemType %s is unknown", itemType) -} - -// GetInstalledItems returns the list of installed items -func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { - items, ok := h.Items[itemType] - if !ok { - return nil, fmt.Errorf("no %s in hubIdx", itemType) - } - - retItems := make([]Item, 0) - - for _, item := range items { - if item.Installed { - retItems = append(retItems, item) - } - } - - return retItems, nil -} - -// GetInstalledItemsAsString returns the names of the installed items -func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { - items, err := h.GetInstalledItems(itemType) - if err != nil { - return nil, err - } - - retStr := make([]string, len(items)) - - for i, it := range items { - retStr[i] = it.Name - } - - return retStr, nil -} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 216a0df4bbd..b51c3aa6ec4 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -9,11 +9,8 @@ import ( "testing" log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) @@ -28,81 +25,6 @@ import ( var responseByPath map[string]string -func TestItemStatus(t *testing.T) { - hub := envSetup(t) - - // get existing map - x := hub.GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.Installed = true - item.UpToDate = false - item.Local = false - item.Tainted = false - - txt, _ := item.Status() - require.Equal(t, "enabled,update-available", txt) - - item.Installed = false - item.UpToDate = false - item.Local = true - item.Tainted = false - - txt, _ = item.Status() - require.Equal(t, "disabled,local", txt) - } - - err := DisplaySummary() - require.NoError(t, err) -} - -func TestGetters(t *testing.T) { - hub := envSetup(t) - - // get non existing map - empty := hub.GetItemMap("ratata") - require.Nil(t, empty) - - // get existing map - x := hub.GetItemMap(COLLECTIONS) - require.NotEmpty(t, x) - - // Get item : good and bad - for k := range x { - empty := hub.GetItem(COLLECTIONS, k+"nope") - require.Nil(t, empty) - - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - // Add item and get it - item.Name += "nope" - err := hub.AddItem(COLLECTIONS, *item) - require.NoError(t, err) - - newitem := hub.GetItem(COLLECTIONS, item.Name) - require.NotNil(t, newitem) - - err = hub.AddItem("ratata", *item) - cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") - } -} - -func TestIndexDownload(t *testing.T) { - hub := envSetup(t) - - _, err := InitHubUpdate(hub.cfg) - require.NoError(t, err, "failed to download index") - - _, err = GetHub() - require.NoError(t, err, "failed to load hub index") -} - // testHub initializes a temporary hub with an empty json file, optionally updating it func testHub(t *testing.T, update bool) *Hub { tmpDir, err := os.MkdirTemp("", "testhub") @@ -115,13 +37,13 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(hubCfg.HubDir, 0700) + err = os.MkdirAll(hubCfg.HubDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDir, 0700) + err = os.MkdirAll(hubCfg.InstallDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDataDir, 0700) + err = os.MkdirAll(hubCfg.InstallDataDir, 0o700) require.NoError(t, err) index, err := os.Create(hubCfg.HubIndexFile) @@ -148,8 +70,9 @@ func testHub(t *testing.T, update bool) *Hub { return hub } +// envSetup initializes the temporary hub and mocks the http client func envSetup(t *testing.T) *Hub { - resetResponseByPath() + setResponseByPath() log.SetLevel(log.DebugLevel) defaultTransport := http.DefaultClient.Transport @@ -163,151 +86,9 @@ func envSetup(t *testing.T) *Hub { hub := testHub(t, true) - // if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - // log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - // } - // if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil { - // log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err) - // } return hub } -func testInstallItem(hub *Hub, t *testing.T, item Item) { - // Install the parser - - err := hub.DownloadLatest(&item, false, false) - require.NoError(t, err, "failed to download %s", item.Name) - - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - err = hub.EnableItem(&item) - require.NoError(t, err, "failed to enable %s", item.Name) - - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) -} - -func testTaintItem(hub *Hub, t *testing.T, item Item) { - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - - f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) - require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) - - defer f.Close() - - _, err = f.WriteString("tainted") - require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) - - // Local sync and check status - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) -} - -func testUpdateItem(hub *Hub, t *testing.T, item Item) { - assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) - - // Update it + check status - err := hub.DownloadLatest(&item, true, true) - require.NoError(t, err, "failed to update %s", item.Name) - - // Local sync and check status - _, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) -} - -func testDisableItem(hub *Hub, t *testing.T, item Item) { - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) - - // Remove - err := hub.DisableItem(&item, false, false) - require.NoError(t, err, "failed to disable %s", item.Name) - - // Local sync and check status - warns, err := hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) - - // Purge - err = hub.DisableItem(&item, true, false) - require.NoError(t, err, "failed to purge %s", item.Name) - - // Local sync and check status - warns, err = hub.LocalSync() - require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) - - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) -} - -func TestInstallParser(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - hub := envSetup(t) - - // map iteration is random by itself - for _, it := range hub.Items[PARSERS] { - testInstallItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testTaintItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testUpdateItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - testDisableItem(hub, t, it) - it = hub.Items[PARSERS][it.Name] - - break - } -} - -func TestInstallCollection(t *testing.T) { - /* - - install a random parser - - check its status - - taint it - - check its status - - force update it - - check its status - - remove it - */ - hub := envSetup(t) - - // map iteration is random by itself - for _, it := range hub.Items[COLLECTIONS] { - testInstallItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testTaintItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testUpdateItem(hub, t, it) - it = hub.Items[COLLECTIONS][it.Name] - testDisableItem(hub, t, it) - break - } -} - type mockTransport struct{} func newMockTransport() http.RoundTripper { @@ -352,7 +133,7 @@ func fileToStringX(path string) string { return strings.ReplaceAll(string(data), "\r\n", "\n") } -func resetResponseByPath() { +func setResponseByPath() { responseByPath = map[string]string{ "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 2255d40a760..e031fde3a93 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,6 +1,7 @@ package cwhub import ( + "errors" "fmt" "io" "net/http" @@ -8,6 +9,7 @@ import ( "path/filepath" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -39,7 +41,7 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) } - file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return err } @@ -70,3 +72,40 @@ func GetData(data []*types.DataSource, dataDir string) error { return nil } + +// downloadData downloads the data files for an item +func downloadData(dataFolder string, force bool, reader io.Reader) error { + var err error + + dec := yaml.NewDecoder(reader) + + for { + data := &DataSet{} + + err = dec.Decode(data) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return fmt.Errorf("while reading file: %w", err) + } + + download := false + + for _, dataS := range data.Data { + if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { + download = true + } + } + + if download || force { + err = GetData(data.Data, dataFolder) + if err != nil { + return fmt.Errorf("while getting data: %w", err) + } + } + } + + return nil +} diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index 40f6ba84790..a072fbb1c2d 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -14,12 +14,14 @@ func TestDownloadFile(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() + //OK httpmock.RegisterResponder( "GET", "https://example.com/xx", httpmock.NewStringResponder(200, "example content oneoneone"), ) + httpmock.RegisterResponder( "GET", "https://example.com/x", @@ -28,15 +30,19 @@ func TestDownloadFile(t *testing.T) { err := downloadFile("https://example.com/xx", examplePath) assert.NoError(t, err) + content, err := os.ReadFile(examplePath) assert.Equal(t, "example content oneoneone", string(content)) assert.NoError(t, err) + //bad uri err = downloadFile("https://zz.com", examplePath) assert.Error(t, err) + //404 err = downloadFile("https://example.com/x", examplePath) assert.Error(t, err) + //bad target err = downloadFile("https://example.com/xx", "") assert.Error(t, err) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go deleted file mode 100644 index 0f4a8fb7a7a..00000000000 --- a/pkg/cwhub/download.go +++ /dev/null @@ -1,336 +0,0 @@ -package cwhub - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" -) - -var ErrIndexNotFound = fmt.Errorf("index not found") - -// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk -// It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { - if cfg == nil { - return nil, fmt.Errorf("no configuration found for hub") - } - - bidx, err := DownloadHubIdx(cfg.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("failed to download index: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("failed to read index: %w", err) - } - } - - theHub = &Hub{ - Items: ret, - cfg: cfg, - } - - if _, err := theHub.LocalSync(); err != nil { - return nil, fmt.Errorf("failed to sync: %w", err) - } - - return theHub, nil -} - -// DownloadHubIdx downloads the latest version of the index and returns the content -func DownloadHubIdx(indexPath string) ([]byte, error) { - log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) - if err != nil { - return nil, fmt.Errorf("failed to build request for hub index: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed http request for hub index: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, ErrIndexNotFound - } - - return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) - } - - oldContent, err := os.ReadFile(indexPath) - if err != nil { - if !os.IsNotExist(err) { - log.Warningf("failed to read hub index: %s", err) - } - } else if bytes.Equal(body, oldContent) { - log.Info("hub index is up to date") - // write it anyway, can't hurt - } - - file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - - if err != nil { - return nil, fmt.Errorf("while opening hub index file: %w", err) - } - defer file.Close() - - wsize, err := file.Write(body) - if err != nil { - return nil, fmt.Errorf("while writing hub index file: %w", err) - } - - log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) - - return body, nil -} - -// DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { - var err error - - log.Debugf("Downloading %s %s", target.Type, target.Name) - - if target.Type != COLLECTIONS { - if !target.Installed && updateOnly && target.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - return nil - } - - return h.DownloadItem(target, overwrite) - } - - // collection - var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} - for idx, ptr := range tmp { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := h.Items[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) - continue - } - - log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly) - //recurse as it's a collection - if ptrtype == COLLECTIONS { - log.Tracef("collection, recurse") - - err = h.DownloadLatest(&val, overwrite, updateOnly) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - } - - downloaded := val.Downloaded - - err = h.DownloadItem(&val, overwrite) - if err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) - } - - // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if val.Downloaded is false because maybe the item has been disabled by the user. - if !val.Installed && !downloaded { - if err = h.EnableItem(&val); err != nil { - return fmt.Errorf("enabling '%s': %w", val.Name, err) - } - } - - h.Items[ptrtype][p] = val - } - } - - err = h.DownloadItem(target, overwrite) - if err != nil { - return fmt.Errorf("failed to download item: %w", err) - } - - return nil -} - -func (h *Hub) DownloadItem(target *Item, overwrite bool) error { - tdir := h.cfg.HubDir - - // if user didn't --force, don't overwrite local, tainted, up-to-date files - if !overwrite { - if target.Tainted { - log.Debugf("%s : tainted, not updated", target.Name) - return nil - } - - if target.UpToDate { - // We still have to check if data files are present - log.Debugf("%s : up-to-date, not updated", target.Name) - } - } - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) - } - - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("while reading %s: %w", req.URL.String(), err) - } - - hash := sha256.New() - if _, err = hash.Write(body); err != nil { - return fmt.Errorf("while hashing %s: %w", target.Name, err) - } - - meow := fmt.Sprintf("%x", hash.Sum(nil)) - if meow != target.Versions[target.Version].Digest { - log.Errorf("Downloaded version doesn't match index, please 'hub update'") - log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) - - return fmt.Errorf("invalid download hash for %s", target.Name) - } - - //all good, install - //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") - parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - - // ensure that target file is within target dir - finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) - } - - if !strings.HasPrefix(finalPath, tdir) { - return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) - } - - // check dir - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Debugf("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating parent directories: %w", err) - } - } - - // check actual file - if _, err = os.Stat(finalPath); !os.IsNotExist(err) { - log.Warningf("%s : overwrite", target.Name) - log.Debugf("target: %s/%s", tdir, target.RemotePath) - } else { - log.Infof("%s : OK", target.Name) - } - - f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("while opening file: %w", err) - } - - defer f.Close() - - _, err = f.Write(body) - if err != nil { - return fmt.Errorf("while writing file: %w", err) - } - - target.Downloaded = true - target.Tainted = false - target.UpToDate = true - - if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { - return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) - } - - h.Items[target.Type][target.Name] = *target - - return nil -} - -// DownloadDataIfNeeded downloads the data files for an item -func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) - - itemFile, err := os.Open(itemFilePath) - if err != nil { - return fmt.Errorf("while opening %s: %w", itemFilePath, err) - } - - defer itemFile.Close() - - if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { - return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) - } - - return nil -} - -// downloadData downloads the data files for an item -func downloadData(dataFolder string, force bool, reader io.Reader) error { - var err error - - dec := yaml.NewDecoder(reader) - - for { - data := &DataSet{} - - err = dec.Decode(data) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return fmt.Errorf("while reading file: %w", err) - } - - download := false - - for _, dataS := range data.Data { - if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { - download = true - } - } - - if download || force { - err = GetData(data.Data, dataFolder) - if err != nil { - return fmt.Errorf("while getting data: %w", err) - } - } - } - - return nil -} diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go deleted file mode 100644 index ed0763081df..00000000000 --- a/pkg/cwhub/download_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package cwhub - -import ( - "fmt" - "os" - "strings" - "testing" - - log "github.com/sirupsen/logrus" -) - -func TestDownloadHubIdx(t *testing.T) { - back := RawFileURLTemplate - // bad url template - fmt.Println("Test 'bad URL'") - - tmpIndex, err := os.CreateTemp("", "index.json") - if err != nil { - t.Fatalf("failed to create temp file : %s", err) - } - - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) - - RawFileURLTemplate = "x" - - ret, err := DownloadHubIdx(tmpIndex.Name()) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad domain - fmt.Println("Test 'bad domain'") - - RawFileURLTemplate = "https://baddomain/%s/%s" - - ret, err = DownloadHubIdx(tmpIndex.Name()) - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { - log.Errorf("unexpected error %s", err) - } - - fmt.Printf("->%+v", ret) - - // bad target path - fmt.Println("Test 'bad target path'") - - RawFileURLTemplate = back - - ret, err = DownloadHubIdx("/does/not/exist/index.json") - if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { - log.Errorf("unexpected error %s", err) - } - - RawFileURLTemplate = back - - fmt.Printf("->%+v", ret) -} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/enable.go similarity index 78% rename from pkg/cwhub/install.go rename to pkg/cwhub/enable.go index 71fb46b1b97..4aac75927b1 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/enable.go @@ -1,5 +1,8 @@ package cwhub +// Enable/disable items already installed (no downloading here) +// This file is not named install.go to avoid confusion with the functions in helpers.go + import ( "fmt" "os" @@ -8,6 +11,81 @@ import ( log "github.com/sirupsen/logrus" ) +// creates symlink between actual config file at hub.HubDir and hub.ConfigDir +// Handles collections recursively +func (h *Hub) EnableItem(target *Item) error { + var err error + + parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") + + // create directories if needed + if target.Installed { + if target.Tainted { + return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) + } + + if target.Local { + return fmt.Errorf("%s is local, won't enable", target.Name) + } + + // if it's a collection, check sub-items even if the collection file itself is up-to-date + if target.UpToDate && target.Type != COLLECTIONS { + log.Tracef("%s is installed and up-to-date, skip.", target.Name) + return nil + } + } + + if _, err = os.Stat(parentDir); os.IsNotExist(err) { + log.Infof("%s doesn't exist, create", parentDir) + + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating directory: %w", err) + } + } + + // install sub-items if it's a collection + if target.Type == COLLECTIONS { + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + } + + err = h.EnableItem(&val) + if err != nil { + return fmt.Errorf("while installing %s: %w", sub.Name, err) + } + } + } + + // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) + if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { + log.Infof("%s already exists.", parentDir+"/"+target.FileName) + return nil + } + + // hub.ConfigDir + target.RemotePath + srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) + if err != nil { + return fmt.Errorf("while getting source path: %w", err) + } + + dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) + if err != nil { + return fmt.Errorf("while getting destination path: %w", err) + } + + if err = os.Symlink(srcPath, dstPath); err != nil { + return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) + } + + log.Infof("Enabled %s : %s", target.Type, target.Name) + target.Installed = true + h.Items[target.Type][target.Name] = *target + + return nil +} + func (h *Hub) purgeItem(target Item) (Item, error) { itempath := h.cfg.HubDir + "/" + target.RemotePath @@ -49,31 +127,30 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { // for a COLLECTIONS, disable sub-items if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if val, ok := h.Items[ptrtype][p]; ok { - // check if the item doesn't belong to another collection before removing it - toRemove := true - - for _, collection := range val.BelongsToCollections { - if collection != target.Name { - toRemove = false - break - } - } - - if toRemove { - err = h.DisableItem(&val, purge, force) - if err != nil { - return fmt.Errorf("while disabling %s: %w", p, err) - } - } else { - log.Infof("%s was not removed because it belongs to another collection", val.Name) - } - } else { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name) + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) + continue + } + + // check if the item doesn't belong to another collection before removing it + toRemove := true + + for _, collection := range val.BelongsToCollections { + if collection != target.Name { + toRemove = false + break + } + } + + if toRemove { + err = h.DisableItem(&val, purge, force) + if err != nil { + return fmt.Errorf("while disabling %s: %w", sub.Name, err) } + } else { + log.Infof("%s was not removed because it belongs to another collection", val.Name) } } } @@ -132,81 +209,3 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return nil } - -// creates symlink between actual config file at hub.HubDir and hub.ConfigDir -// Handles collections recursively -func (h *Hub) EnableItem(target *Item) error { - var err error - - parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") - - // create directories if needed - if target.Installed { - if target.Tainted { - return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) - } - - if target.Local { - return fmt.Errorf("%s is local, won't enable", target.Name) - } - - // if it's a collection, check sub-items even if the collection file itself is up-to-date - if target.UpToDate && target.Type != COLLECTIONS { - log.Tracef("%s is installed and up-to-date, skip.", target.Name) - return nil - } - } - - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Infof("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating directory: %w", err) - } - } - - // install sub-items if it's a collection - if target.Type == COLLECTIONS { - for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - val, ok := h.Items[ptrtype][p] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) - } - - err = h.EnableItem(&val) - if err != nil { - return fmt.Errorf("while installing %s: %w", p, err) - } - } - } - } - - // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { - log.Infof("%s already exists.", parentDir+"/"+target.FileName) - return nil - } - - // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) - if err != nil { - return fmt.Errorf("while getting source path: %w", err) - } - - dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) - if err != nil { - return fmt.Errorf("while getting destination path: %w", err) - } - - if err = os.Symlink(srcPath, dstPath); err != nil { - return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) - } - - log.Infof("Enabled %s : %s", target.Type, target.Name) - target.Installed = true - h.Items[target.Type][target.Name] = *target - - return nil -} diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go new file mode 100644 index 00000000000..864b3fa8587 --- /dev/null +++ b/pkg/cwhub/enable_test.go @@ -0,0 +1,144 @@ +package cwhub + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testInstall(hub *Hub, t *testing.T, item Item) { + // Install the parser + err := hub.DownloadLatest(&item, false, false) + require.NoError(t, err, "failed to download %s", item.Name) + + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + + err = hub.EnableItem(&item) + require.NoError(t, err, "failed to enable %s", item.Name) + + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) +} + +func testTaint(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + + f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) + require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) + + defer f.Close() + + _, err = f.WriteString("tainted") + require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) + + // Local sync and check status + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) +} + +func testUpdate(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) + + // Update it + check status + err := hub.DownloadLatest(&item, true, true) + require.NoError(t, err, "failed to update %s", item.Name) + + // Local sync and check status + _, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) +} + +func testDisable(hub *Hub, t *testing.T, item Item) { + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + + // Remove + err := hub.DisableItem(&item, false, false) + require.NoError(t, err, "failed to disable %s", item.Name) + + // Local sync and check status + warns, err := hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, warns, "unexpected warnings : %+v", warns) + + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + + // Purge + err = hub.DisableItem(&item, true, false) + require.NoError(t, err, "failed to purge %s", item.Name) + + // Local sync and check status + warns, err = hub.LocalSync() + require.NoError(t, err, "failed to run localSync") + require.Empty(t, warns, "unexpected warnings : %+v", warns) + + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) +} + +func TestInstallParser(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[PARSERS] { + testInstall(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testTaint(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testDisable(hub, t, it) + + break + } +} + +func TestInstallCollection(t *testing.T) { + /* + - install a random parser + - check its status + - taint it + - check its status + - force update it + - check its status + - remove it + */ + hub := envSetup(t) + + // map iteration is random by itself + for _, it := range hub.Items[COLLECTIONS] { + testInstall(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testTaint(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testUpdate(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testDisable(hub, t, it) + + break + } +} diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 05c34824d40..ad66336be41 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -1,65 +1,23 @@ package cwhub +// Install, upgrade and remove items from the hub to the local configuration + +// XXX: this file could use a better name + import ( + "bytes" + "crypto/sha256" "fmt" + "io" + "net/http" + "os" "path/filepath" + "strings" "github.com/enescakir/emoji" log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) -// chooseHubBranch returns the branch name to use for the hub -// It can be "master" or branch corresponding to the current crowdsec version -func chooseHubBranch() string { - latest, err := cwversion.Latest() - if err != nil { - log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - return "master" - } - - csVersion := cwversion.VersionStrip() - if csVersion == latest { - log.Debugf("current version is equal to latest (%s)", csVersion) - return "master" - } - - // if current version is greater than the latest we are in pre-release - if semver.Compare(csVersion, latest) == 1 { - log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - return "master" - } - - if csVersion == "" { - log.Warning("Crowdsec version is not set, using master branch for the hub") - return "master" - } - - log.Warnf("Crowdsec is not the latest version. "+ - "Current version is '%s' and the latest stable version is '%s'. Please update it!", - csVersion, latest) - - log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ - "added to Crowdsec Hub after CrowdSec %s", latest) - - return csVersion -} - -// SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() { - // a branch is already set, or specified from the flags - if HubBranch != "" { - return - } - - // use the branch corresponding to the crowdsec version - HubBranch = chooseHubBranch() - - log.Debugf("Using branch '%s' for the hub", HubBranch) -} - // InstallItem installs an item from the hub func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { item := h.GetItem(itemType, name) @@ -80,7 +38,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = h.AddItem(itemType, *item); err != nil { + if err = h.AddItem(*item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -94,7 +52,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := h.AddItem(itemType, *item); err != nil { + if err := h.AddItem(*item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -117,7 +75,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for return fmt.Errorf("unable to disable %s: %w", item.Name, err) } - if err = h.AddItem(itemType, *item); err != nil { + if err = h.AddItem(*item); err != nil { return fmt.Errorf("unable to add %s: %w", item.Name, err) } @@ -141,7 +99,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for return fmt.Errorf("unable to disable %s: %w", v.Name, err) } - if err := h.AddItem(itemType, v); err != nil { + if err := h.AddItem(v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ @@ -204,7 +162,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { updated++ } - if err := h.AddItem(itemType, v); err != nil { + if err := h.AddItem(v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } } @@ -225,3 +183,192 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { return nil } + +// DownloadLatest will download the latest version of Item to the tdir directory +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { + var err error + + log.Debugf("Downloading %s %s", target.Type, target.Name) + + if target.Type != COLLECTIONS { + if !target.Installed && updateOnly && target.Downloaded { + log.Debugf("skipping upgrade of %s : not installed", target.Name) + return nil + } + + return h.DownloadItem(target, overwrite) + } + + // collection + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + } + + if !val.Installed && updateOnly && val.Downloaded { + log.Debugf("skipping upgrade of %s : not installed", target.Name) + continue + } + + log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) + //recurse as it's a collection + if sub.Type == COLLECTIONS { + log.Tracef("collection, recurse") + + err = h.DownloadLatest(&val, overwrite, updateOnly) + if err != nil { + return fmt.Errorf("while downloading %s: %w", val.Name, err) + } + } + + downloaded := val.Downloaded + + err = h.DownloadItem(&val, overwrite) + if err != nil { + return fmt.Errorf("while downloading %s: %w", val.Name, err) + } + + // We need to enable an item when it has been added to a collection since latest release of the collection. + // We check if val.Downloaded is false because maybe the item has been disabled by the user. + if !val.Installed && !downloaded { + if err = h.EnableItem(&val); err != nil { + return fmt.Errorf("enabling '%s': %w", val.Name, err) + } + } + + h.Items[sub.Type][sub.Name] = val + } + + err = h.DownloadItem(target, overwrite) + if err != nil { + return fmt.Errorf("failed to download item: %w", err) + } + + return nil +} + +func (h *Hub) DownloadItem(target *Item, overwrite bool) error { + tdir := h.cfg.HubDir + + // if user didn't --force, don't overwrite local, tainted, up-to-date files + if !overwrite { + if target.Tainted { + log.Debugf("%s : tainted, not updated", target.Name) + return nil + } + + if target.UpToDate { + // We still have to check if data files are present + log.Debugf("%s : up-to-date, not updated", target.Name) + } + } + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) + if err != nil { + return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("while reading %s: %w", req.URL.String(), err) + } + + hash := sha256.New() + if _, err = hash.Write(body); err != nil { + return fmt.Errorf("while hashing %s: %w", target.Name, err) + } + + meow := fmt.Sprintf("%x", hash.Sum(nil)) + if meow != target.Versions[target.Version].Digest { + log.Errorf("Downloaded version doesn't match index, please 'hub update'") + log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) + + return fmt.Errorf("invalid download hash for %s", target.Name) + } + + //all good, install + //check if parent dir exists + tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") + parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") + + // ensure that target file is within target dir + finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) + if err != nil { + return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) + } + + if !strings.HasPrefix(finalPath, tdir) { + return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) + } + + // check dir + if _, err = os.Stat(parentDir); os.IsNotExist(err) { + log.Debugf("%s doesn't exist, create", parentDir) + + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating parent directories: %w", err) + } + } + + // check actual file + if _, err = os.Stat(finalPath); !os.IsNotExist(err) { + log.Warningf("%s : overwrite", target.Name) + log.Debugf("target: %s/%s", tdir, target.RemotePath) + } else { + log.Infof("%s : OK", target.Name) + } + + f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("while opening file: %w", err) + } + + defer f.Close() + + _, err = f.Write(body) + if err != nil { + return fmt.Errorf("while writing file: %w", err) + } + + target.Downloaded = true + target.Tainted = false + target.UpToDate = true + + if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) + } + + h.Items[target.Type][target.Name] = *target + + return nil +} + +// DownloadDataIfNeeded downloads the data files for an item +func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) + + itemFile, err := os.Open(itemFilePath) + if err != nil { + return fmt.Errorf("while opening %s: %w", itemFilePath, err) + } + + defer itemFile.Close() + + if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { + return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) + } + + return nil +} diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 72dac2342a2..1ece43269ac 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -14,8 +14,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -59,8 +57,6 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) @@ -99,6 +95,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { hub, err := InitHub(hubCfg) require.NoError(t, err, "failed to load hub index") + return hub } @@ -109,8 +106,6 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * hub := envSetup(t) // fresh install of collection - hub = getHubOrFail(t, hub.cfg) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 6c9dce94b4d..2a2bfbb0a0b 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -1,8 +1,13 @@ package cwhub import ( + "bytes" "encoding/json" + "errors" "fmt" + "io" + "net/http" + "os" "strings" log "github.com/sirupsen/logrus" @@ -10,22 +15,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -const ( - HubIndexFile = ".index.json" - - // managed item types - COLLECTIONS = "collections" - PARSERS = "parsers" - POSTOVERFLOWS = "postoverflows" - SCENARIOS = "scenarios" -) - -var ( - // XXX: The order is important, as it is used to range over sub-items in collections - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} -) - -type HubItems map[string]map[string]Item +const HubIndexFile = ".index.json" // Hub represents the runtime status of the hub (parsed items, etc.) type Hub struct { @@ -35,7 +25,10 @@ type Hub struct { skippedTainted int } -var theHub *Hub +var ( + theHub *Hub + ErrIndexNotFound = fmt.Errorf("index not found") +) // GetHub returns the hub singleton // it returns an error if it's not initialized to avoid nil dereference @@ -47,32 +40,129 @@ func GetHub() (*Hub, error) { return theHub, nil } -// displaySummary prints a total count of the hub items -func (h Hub) displaySummary() { - msg := "Loaded: " - for itemType := range h.Items { - msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) +// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use +func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") } - log.Info(strings.Trim(msg, ", ")) - if h.skippedLocal > 0 || h.skippedTainted > 0 { - log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted) + log.Debugf("loading hub idx %s", cfg.HubIndexFile) + + bidx, err := os.ReadFile(cfg.HubIndexFile) + if err != nil { + return nil, fmt.Errorf("unable to read index file: %w", err) + } + + ret, err := ParseIndex(bidx) + if err != nil { + if !errors.Is(err, ErrMissingReference) { + return nil, fmt.Errorf("unable to load existing index: %w", err) + } + + // XXX: why the error check if we bail out anyway? + return nil, err + } + + theHub = &Hub{ + Items: ret, + cfg: cfg, } + + _, err = theHub.LocalSync() + if err != nil { + return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) + } + + return theHub, nil } -// DisplaySummary prints a total count of the hub items. -// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton -// XXX: to be removed later -func DisplaySummary() error { - hub, err := GetHub() +// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk +// It is used to inizialize the hub when there is no index file yet +func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") + } + + bidx, err := DownloadIndex(cfg.HubIndexFile) if err != nil { - return err + return nil, fmt.Errorf("failed to download index: %w", err) } - hub.displaySummary() - return nil + + ret, err := ParseIndex(bidx) + if err != nil { + if !errors.Is(err, ErrMissingReference) { + return nil, fmt.Errorf("failed to read index: %w", err) + } + } + + theHub = &Hub{ + Items: ret, + cfg: cfg, + } + + if _, err := theHub.LocalSync(); err != nil { + return nil, fmt.Errorf("failed to sync: %w", err) + } + + return theHub, nil } -// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections +// DownloadIndex downloads the latest version of the index and returns the content +func DownloadIndex(indexPath string) ([]byte, error) { + log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) + if err != nil { + return nil, fmt.Errorf("failed to build request for hub index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed http request for hub index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, ErrIndexNotFound + } + + return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) + } + + oldContent, err := os.ReadFile(indexPath) + if err != nil { + if !os.IsNotExist(err) { + log.Warningf("failed to read hub index: %s", err) + } + } else if bytes.Equal(body, oldContent) { + log.Info("hub index is up to date") + // write it anyway, can't hurt + } + + file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + + if err != nil { + return nil, fmt.Errorf("while opening hub index file: %w", err) + } + defer file.Close() + + wsize, err := file.Write(body) + if err != nil { + return nil, fmt.Errorf("while writing hub index file: %w", err) + } + + log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) + + return body, nil +} + +// ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections func ParseIndex(buff []byte) (HubItems, error) { var ( RawIndex HubItems @@ -102,13 +192,10 @@ func ParseIndex(buff []byte) (HubItems, error) { // if it's a collection, check its sub-items are present // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } + for _, sub := range item.SubItems() { + if _, ok := RawIndex[sub.Type][sub.Name]; !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name) + missingItems = append(missingItems, sub.Name) } } } @@ -120,3 +207,34 @@ func ParseIndex(buff []byte) (HubItems, error) { return RawIndex, nil } + +// ItemStats returns total counts of the hub items +func (h Hub) ItemStats() []string { + loaded := "" + for _, itemType := range ItemTypes { + // ensure the order is always the same + if h.Items[itemType] == nil { + continue + } + if len(h.Items[itemType]) == 0 { + continue + } + loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + } + + loaded = strings.Trim(loaded, ", ") + if loaded == "" { + // empty hub + loaded = "0 items" + } + + ret := []string{ + fmt.Sprintf("Loaded: %s", loaded), + } + + if h.skippedLocal > 0 || h.skippedTainted > 0 { + ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)) + } + + return ret +} diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go new file mode 100644 index 00000000000..1b3a31109dd --- /dev/null +++ b/pkg/cwhub/hub_test.go @@ -0,0 +1,63 @@ +package cwhub + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" +) + +func TestInitHubUpdate(t *testing.T) { + hub := envSetup(t) + + _, err := InitHubUpdate(hub.cfg) + require.NoError(t, err) + + _, err = GetHub() + require.NoError(t, err) +} + +func TestDownloadIndex(t *testing.T) { + back := RawFileURLTemplate + // bad url template + fmt.Println("Test 'bad URL'") + + tmpIndex, err := os.CreateTemp("", "index.json") + require.NoError(t, err) + + t.Cleanup(func() { + os.Remove(tmpIndex.Name()) + }) + + RawFileURLTemplate = "x" + + ret, err := DownloadIndex(tmpIndex.Name()) + cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ") + + fmt.Printf("->%+v", ret) + + // bad domain + fmt.Println("Test 'bad domain'") + + RawFileURLTemplate = "https://baddomain/%s/%s" + + ret, err = DownloadIndex(tmpIndex.Name()) + cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") + + fmt.Printf("->%+v", ret) + + // bad target path + fmt.Println("Test 'bad target path'") + + RawFileURLTemplate = back + + ret, err = DownloadIndex("/does/not/exist/index.json") + cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") + + RawFileURLTemplate = back + + fmt.Printf("->%+v", ret) +} diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go new file mode 100644 index 00000000000..7d20b107e12 --- /dev/null +++ b/pkg/cwhub/items.go @@ -0,0 +1,232 @@ +package cwhub + +import ( + "fmt" + + "github.com/enescakir/emoji" + "golang.org/x/mod/semver" +) + +const ( + // managed item types + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" +) + +// XXX: The order is important, as it is used to range over sub-items in collections +var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} + +type HubItems map[string]map[string]Item + +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted. +type ItemVersion struct { + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? +} + +// Item represents an object managed in the hub. It can be a parser, scenario, collection.. +type Item struct { + // descriptive info + Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" + FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any + + // remote (hub) info + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml + Version string `json:"version,omitempty"` // the last version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions + + // local (deployed) info + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` // the local meow + Installed bool `json:"installed,omitempty"` + Downloaded bool `json:"downloaded,omitempty"` + UpToDate bool `json:"up_to_date,omitempty"` + Tainted bool `json:"tainted,omitempty"` // has it been locally modified? + Local bool `json:"local,omitempty"` // if it's a non versioned control one + + // if it's a collection, it can have sub items + Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` + PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` + Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` + Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` +} + + +type SubItem struct { + Type string + Name string +} + +func (i *Item) SubItems() []SubItem { + sub := make([]SubItem, + len(i.Parsers) + + len(i.PostOverflows) + + len(i.Scenarios) + + len(i.Collections)) + n := 0 + for _, name := range i.Parsers { + sub[n] = SubItem{Type: PARSERS, Name: name} + n++ + } + for _, name := range i.PostOverflows { + sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name} + n++ + } + for _, name := range i.Scenarios { + sub[n] = SubItem{Type: SCENARIOS, Name: name} + n++ + } + for _, name := range i.Collections { + sub[n] = SubItem{Type: COLLECTIONS, Name: name} + n++ + } + return sub +} + +// Status returns the status of the item as a string and an emoji +// ie. "enabled,update-available" and emoji.Warning +func (i *Item) Status() (string, emoji.Emoji) { + status := "disabled" + ok := false + + if i.Installed { + ok = true + status = "enabled" + } + + managed := true + if i.Local { + managed = false + status += ",local" + } + + warning := false + if i.Tainted { + warning = true + status += ",tainted" + } else if !i.UpToDate && !i.Local { + warning = true + status += ",update-available" + } + + emo := emoji.QuestionMark + + switch { + case !managed: + emo = emoji.House + case !i.Installed: + emo = emoji.Prohibited + case warning: + emo = emoji.Warning + case ok: + emo = emoji.CheckMark + } + + return status, emo +} + +// versionStatus: semver requires 'v' prefix +func (i *Item) versionStatus() int { + return semver.Compare("v"+i.Version, "v"+i.LocalVersion) +} + +// validPath returns true if the (relative) path is allowed for the item +// dirNmae: the directory name (ie. crowdsecurity) +// fileName: the filename (ie. apache2-logs.yaml) +func (i *Item) validPath(dirName, fileName string) bool { + return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") +} + +// GetItemMap returns the map of items for a given type +func (h *Hub) GetItemMap(itemType string) map[string]Item { + m, ok := h.Items[itemType] + if !ok { + return nil + } + + return m +} + +// GetItem returns the item from hub based on its type and full name (author/name) +func (h *Hub) GetItem(itemType string, itemName string) *Item { + m, ok := h.GetItemMap(itemType)[itemName] + if !ok { + return nil + } + + return &m +} + +// GetItemNames returns the list of item (full) names for a given type +// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// The names can be used to retrieve the item with GetItem() +func (h *Hub) GetItemNames(itemType string) []string { + m := h.GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + +// AddItem adds an item to the hub index +func (h *Hub) AddItem(item Item) error { + for _, t := range ItemTypes { + if t == item.Type { + h.Items[t][item.Name] = item + return nil + } + } + + return fmt.Errorf("ItemType %s is unknown", item.Type) +} + +// GetInstalledItems returns the list of installed items +func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { + items, ok := h.Items[itemType] + if !ok { + return nil, fmt.Errorf("no %s in hubIdx", itemType) + } + + retItems := make([]Item, 0) + + for _, item := range items { + if item.Installed { + retItems = append(retItems, item) + } + } + + return retItems, nil +} + +// GetInstalledItemsAsString returns the names of the installed items +func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { + items, err := h.GetInstalledItems(itemType) + if err != nil { + return nil, err + } + + retStr := make([]string, len(items)) + + for i, it := range items { + retStr[i] = it.Name + } + + return retStr, nil +} diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go new file mode 100644 index 00000000000..e16dbe31200 --- /dev/null +++ b/pkg/cwhub/items_test.go @@ -0,0 +1,75 @@ +package cwhub + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" +) + +func TestItemStatus(t *testing.T) { + hub := envSetup(t) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item : good and bad + for k := range x { + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + item.Installed = true + item.UpToDate = false + item.Local = false + item.Tainted = false + + txt, _ := item.Status() + require.Equal(t, "enabled,update-available", txt) + + item.Installed = false + item.UpToDate = false + item.Local = true + item.Tainted = false + + txt, _ = item.Status() + require.Equal(t, "disabled,local", txt) + } + + stats := hub.ItemStats() + require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats) +} + +func TestGetters(t *testing.T) { + hub := envSetup(t) + + // get non existing map + empty := hub.GetItemMap("ratata") + require.Nil(t, empty) + + // get existing map + x := hub.GetItemMap(COLLECTIONS) + require.NotEmpty(t, x) + + // Get item : good and bad + for k := range x { + empty := hub.GetItem(COLLECTIONS, k+"nope") + require.Nil(t, empty) + + item := hub.GetItem(COLLECTIONS, k) + require.NotNil(t, item) + + // Add item and get it + item.Name += "nope" + err := hub.AddItem(*item) + require.NoError(t, err) + + newitem := hub.GetItem(COLLECTIONS, item.Name) + require.NotNil(t, newitem) + + item.Type = "ratata" + err = hub.AddItem(*item) + cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") + } +} diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go new file mode 100644 index 00000000000..fe26aa4c107 --- /dev/null +++ b/pkg/cwhub/leakybucket.go @@ -0,0 +1,58 @@ +package cwhub + +// Resolve a symlink to find the hub item it points to. +// This file is used only by pkg/leakybucket + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary +func itemKey(itemPath string) (string, error) { + f, err := os.Lstat(itemPath) + if err != nil { + return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) + } + + if f.Mode()&os.ModeSymlink == 0 { + // it's not a symlink, so the filename itsef should be the key + return filepath.Base(itemPath), nil + } + + // resolve the symlink to hub file + pathInHub, err := os.Readlink(itemPath) + if err != nil { + return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) + } + + author := filepath.Base(filepath.Dir(pathInHub)) + + fname := filepath.Base(pathInHub) + fname = strings.TrimSuffix(fname, ".yaml") + fname = strings.TrimSuffix(fname, ".yml") + + return fmt.Sprintf("%s/%s", author, fname), nil +} + +// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. +func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { + itemKey, err := itemKey(itemPath) + if err != nil { + return nil, err + } + + m := h.GetItemMap(itemType) + if m == nil { + return nil, fmt.Errorf("item type %s doesn't exist", itemType) + } + + v, ok := m[itemKey] + if !ok { + return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) + } + + return &v, nil +} diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/sync.go similarity index 80% rename from pkg/cwhub/loader.go rename to pkg/cwhub/sync.go index 31e347067d5..a9360a93c25 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/sync.go @@ -2,7 +2,6 @@ package cwhub import ( "crypto/sha256" - "errors" "fmt" "io" "os" @@ -11,8 +10,6 @@ import ( "strings" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) func isYAMLFileName(path string) bool { @@ -109,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) // log.Infof("%s -> name:%s stage:%s", path, fname, stage) - + if ret.stage == SCENARIOS { ret.ftype = SCENARIOS ret.stage = "" @@ -325,66 +322,63 @@ func (h *Hub) CollectDepsCheck(v *Item) error { // if it's a collection, ensure all the items are installed, or tag it as tainted log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) - for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { - sliceType := ItemTypes[idx] - for _, subName := range itemSlice { - subItem, ok := h.Items[sliceType][subName] - if !ok { - return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) - } - - log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) + for _, sub := range v.SubItems() { + subItem, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name) + } - if !v.Installed { - continue - } + log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) - if subItem.Type == COLLECTIONS { - log.Tracef("collec, recurse.") + if !v.Installed { + continue + } - if err := h.CollectDepsCheck(&subItem); err != nil { - if subItem.Tainted { - v.Tainted = true - } + if subItem.Type == COLLECTIONS { + log.Tracef("collec, recurse.") - return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) + if err := h.CollectDepsCheck(&subItem); err != nil { + if subItem.Tainted { + v.Tainted = true } - h.Items[sliceType][subName] = subItem + return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - // propagate the state of sub-items to set - if subItem.Tainted { - v.Tainted = true - return fmt.Errorf("tainted %s %s, tainted", sliceType, subName) - } + h.Items[sub.Type][sub.Name] = subItem + } - if !subItem.Installed && v.Installed { - v.Tainted = true - return fmt.Errorf("missing %s %s, tainted", sliceType, subName) - } + // propagate the state of sub-items to set + if subItem.Tainted { + v.Tainted = true + return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) + } - if !subItem.UpToDate { - v.UpToDate = false - return fmt.Errorf("outdated %s %s", sliceType, subName) - } + if !subItem.Installed && v.Installed { + v.Tainted = true + return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) + } - skip := false + if !subItem.UpToDate { + v.UpToDate = false + return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) + } - for idx := range subItem.BelongsToCollections { - if subItem.BelongsToCollections[idx] == v.Name { - skip = true - } - } + skip := false - if !skip { - subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) + for idx := range subItem.BelongsToCollections { + if subItem.BelongsToCollections[idx] == v.Name { + skip = true } + } - h.Items[sliceType][subName] = subItem - - log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) + if !skip { + subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } + + h.Items[sub.Type][sub.Name] = subItem + + log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) } return nil @@ -447,39 +441,3 @@ func (h *Hub) LocalSync() ([]string, error) { return warnings, nil } - -// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use -func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { - if cfg == nil { - return nil, fmt.Errorf("no configuration found for hub") - } - - log.Debugf("loading hub idx %s", cfg.HubIndexFile) - - bidx, err := os.ReadFile(cfg.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("unable to read index file: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("unable to load existing index: %w", err) - } - - // XXX: why the error check if we bail out anyway? - return nil, err - } - - theHub = &Hub{ - Items: ret, - cfg: cfg, - } - - _, err = theHub.LocalSync() - if err != nil { - return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) - } - - return theHub, nil -} diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 5ccbbad39d3..489b324c4b3 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -22,7 +22,7 @@ func Copy(src string, dst string) error { return err } - err = os.WriteFile(dst, content, 0644) + err = os.WriteFile(dst, content, 0o644) if err != nil { return err } diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index db33ae97fb2..694c92f8a93 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -258,6 +258,14 @@ teardown() { assert_output "0" } +@test "cscli parsers remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" +} + @test "cscli collections upgrade [collection]..." { rune -1 cscli collections upgrade assert_stderr --partial "specify at least one collection to upgrade or '--all'" diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index 26844c96586..a703c65968d 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -49,8 +49,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/smb # XXX: should this be an error? rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]" - assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection" + assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/smb]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" rune -0 cscli collections list -o json rune -0 jq -c '[.collections[].name]' <(output) assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]' diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 77c3dc089a9..b349cf4e2e0 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -268,6 +268,14 @@ teardown() { assert_output "0" } +@test "cscli parsers remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli parsers remove crowdsecurity/sshd-logs + assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" +} + @test "cscli parsers upgrade [parser]..." { rune -1 cscli parsers upgrade assert_stderr --partial "specify at least one parser to upgrade or '--all'" diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index cf74268f724..da0f17b05e9 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -260,6 +260,14 @@ teardown() { assert_output "0" } +@test "cscli postoverflows remove [parser]... --force" { + # remove a parser that belongs to a collection + rune -0 cscli collections install crowdsecurity/auditd + rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process + assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]" + assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow" +} + @test "cscli postoverflows upgrade [postoverflow]..." { rune -1 cscli postoverflows upgrade assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 2eeb6146bc2..04c344fe42d 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -260,6 +260,14 @@ teardown() { assert_output "0" } +@test "cscli scenarios remove [scenario]... --force" { + # remove a scenario that belongs to a collection + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf belongs to collections: [crowdsecurity/sshd]" + assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario" +} + @test "cscli scenarios upgrade [scenario]..." { rune -1 cscli scenarios upgrade assert_stderr --partial "specify at least one scenario to upgrade or '--all'" From dca6faab08df8ecad873b9bb1b00497811591c9c Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 23 Oct 2023 10:53:39 +0200 Subject: [PATCH 112/263] logger --- pkg/acquisition/modules/waap/waap.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 8e844e2d8e2..da77b16b459 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -167,23 +167,20 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { wafUUID := uuid.New().String() - wafLogger := &log.Entry{} + var wafLogger *log.Entry //configure logger + var clog = log.New() + if err := types.ConfigureLogger(clog); err != nil { + log.Fatalf("While creating bucket-specific logger : %s", err) + } + //is it still needed ? if w.config.Debug { - var clog = log.New() - if err := types.ConfigureLogger(clog); err != nil { - log.Fatalf("While creating bucket-specific logger : %s", err) - } clog.SetLevel(log.DebugLevel) - wafLogger = clog.WithFields(log.Fields{ - "uuid": wafUUID, - }) - } else { - wafLogger = log.WithFields(log.Fields{ - "uuid": wafUUID, - }) } + wafLogger = clog.WithFields(log.Fields{ + "uuid": wafUUID, + }) //we copy WaapRutime for each runner wrt := *w.WaapRuntime From 2ff238d5f85e391399b1268c4405bf1ac8da9034 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 23 Oct 2023 10:53:52 +0200 Subject: [PATCH 113/263] logger --- pkg/acquisition/modules/waap/waap_runner.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 77ddd0263d7..ed52abebb66 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -40,9 +40,12 @@ func (r *WaapRunner) Init(datadir string) error { for _, collection := range r.WaapRuntime.OutOfBandRules { outOfBandRules += collection.String() } - + //adapt the logger level to the WAAP + runnerLogger := r.logger.Dup() + runnerLogger.Logger.SetLevel(r.WaapRuntime.Config.LogLevel) + runnerLogger.Infof("setting logger of %s to %s", r.WaapRuntime.Name, r.WaapRuntime.Config.LogLevel) r.WaapInbandEngine, err = coraza.NewWAF( - coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs), + coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), ) if err != nil { @@ -50,7 +53,7 @@ func (r *WaapRunner) Init(datadir string) error { } r.WaapOutbandEngine, err = coraza.NewWAF( - coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs), + coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), ) if err != nil { From c00b1abd72e251f712a72a5543ef838f8bf14c78 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 23 Oct 2023 10:54:11 +0200 Subject: [PATCH 114/263] logger --- pkg/waf/waap.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 21f7229325c..4982c5763d2 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -60,6 +60,7 @@ type WaapRuntimeConfig struct { CompiledOnMatch []Hook CompiledVariablesTracking []*regexp.Regexp Config *WaapConfig + //CorazaLogger debuglog.Logger //those are ephemeral, created/destroyed with every req OutOfBandTx ExtendedTransaction //is it a good idea ? @@ -80,6 +81,7 @@ type WaapConfig struct { PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` VariablesTracking []string `yaml:"variables_tracking"` + LogLevel log.Level `yaml:"log_level"` Logger *log.Entry `yaml:"-"` } @@ -104,9 +106,12 @@ func (wc *WaapConfig) Load(file string) error { if err != nil { return fmt.Errorf("unable to parse yaml file %s : %s", file, err) } + if wc.Name == "" { return fmt.Errorf("name cannot be empty") } + wc.Logger = wc.Logger.WithField("name", wc.Name) + wc.Logger.Logger.SetLevel(wc.LogLevel) if wc.DefaultRemediation == "" { return fmt.Errorf("default_remediation cannot be empty") } From 1b9d8c8226e33fbd3d9cec361d69bab10adf38b0 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 23 Oct 2023 10:54:26 +0200 Subject: [PATCH 115/263] logger --- pkg/waf/waap_rules_collection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 14e25e74c9a..6d05245db60 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -43,7 +43,7 @@ func LoadCollection(collection string) (WaapCollection, error) { } for _, hubWafRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { - log.Infof("loading %s", hubWafRuleItem.LocalPath) + //log.Infof("loading %s", hubWafRuleItem.LocalPath) if !hubWafRuleItem.Installed { continue } From bd9df8f480661cf6ad358077bcece482d113e917 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 23 Oct 2023 10:59:02 +0200 Subject: [PATCH 116/263] logger --- pkg/acquisition/modules/waap/coraza_logger.go | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 pkg/acquisition/modules/waap/coraza_logger.go diff --git a/pkg/acquisition/modules/waap/coraza_logger.go b/pkg/acquisition/modules/waap/coraza_logger.go new file mode 100644 index 00000000000..f9a5ee42784 --- /dev/null +++ b/pkg/acquisition/modules/waap/coraza_logger.go @@ -0,0 +1,189 @@ +package wafacquisition + +import ( + "fmt" + "io" + + dbg "github.com/crowdsecurity/coraza/v3/debuglog" + log "github.com/sirupsen/logrus" +) + +// type ContextField func(Event) Event + +type crzLogEvent struct { + fields log.Fields + logger *log.Entry + muted bool +} + +func (e *crzLogEvent) Msg(msg string) { + if e.muted { + return + } + if len(e.fields) == 0 { + e.logger.Info(msg) + } else { + e.logger.WithFields(e.fields).Info(msg) + } +} + +func (e *crzLogEvent) Str(key, val string) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("str") + e.fields[key] = val + return e +} + +func (e *crzLogEvent) Err(err error) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("err") + e.fields["error"] = err + return e +} + +func (e *crzLogEvent) Bool(key string, b bool) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("bool") + e.fields[key] = b + return e +} + +func (e *crzLogEvent) Int(key string, i int) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("int") + e.fields[key] = i + return e +} + +func (e *crzLogEvent) Uint(key string, i uint) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("uint") + e.fields[key] = i + return e +} + +func (e *crzLogEvent) Stringer(key string, val fmt.Stringer) dbg.Event { + if e.muted { + return e + } + //e.logger.Info("stringer") + e.fields[key] = val + return e +} + +func (e crzLogEvent) IsEnabled() bool { + if e.muted { + return false + } + return true +} + +type crzLogger struct { + logger *log.Entry + defaultFields log.Fields + logLevel log.Level +} + +func NewCrzLogger(logger *log.Entry) crzLogger { + return crzLogger{logger: logger, logLevel: logger.Logger.GetLevel()} +} + +func (c crzLogger) WithOutput(w io.Writer) dbg.Logger { + c.logger.Infof("ignoring withoutput directive") + return c +} + +func (c crzLogger) WithLevel(lvl dbg.Level) dbg.Logger { + c.logger.Warningf("setting log level to %s", lvl) + c.logLevel = log.Level(lvl) + c.logger.Logger.SetLevel(c.logLevel) + return c +} + +func (c crzLogger) With(fs ...dbg.ContextField) dbg.Logger { + var e dbg.Event = &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + for _, f := range fs { + e = f(e) + } + c.defaultFields = e.(*crzLogEvent).fields + return c +} + +func (c crzLogger) Trace() dbg.Event { + if c.logLevel < log.TraceLevel { + //c.logger.Infof("ignoring trace directive") + return &crzLogEvent{muted: true} + } + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + +func (c crzLogger) Debug() dbg.Event { + if c.logLevel < log.DebugLevel { + //c.logger.Infof("ignoring debug directive -> %s", c.logLevel.String()) + return &crzLogEvent{muted: true} + + } + + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + +func (c crzLogger) Info() dbg.Event { + if c.logLevel < log.InfoLevel { + return &crzLogEvent{muted: true} + } + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + +func (c crzLogger) Warn() dbg.Event { + if c.logLevel < log.WarnLevel { + return &crzLogEvent{muted: true} + } + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + +func (c crzLogger) Error() dbg.Event { + if c.logLevel < log.ErrorLevel { + return &crzLogEvent{muted: true} + } + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} From 00e1ffbf585d91aac8a85e6aee043937788da6d0 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 10:49:28 +0200 Subject: [PATCH 117/263] simplify a bit --- pkg/acquisition/modules/waap/waap.go | 34 +++++----------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index da77b16b459..272c50536a1 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" - corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -33,7 +32,6 @@ type WaapSourceConfig struct { KeyFilePath string `yaml:"key_file"` Path string `yaml:"path"` Routines int `yaml:"routines"` - Debug bool `yaml:"debug"` WaapConfig string `yaml:"waap_config"` WaapConfigPath string `yaml:"waap_config_path"` configuration.DataSourceCommonCfg `yaml:",inline"` @@ -106,19 +104,13 @@ func (w *WaapSource) GetAggregMetrics() []prometheus.Collector { return nil } -func logError(error corazatypes.MatchedRule) { - msg := error.ErrorLog() - log.Infof("[logError][%s] %s", error.Rule().Severity(), msg) -} - func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { - //wc := WaapSourceConfig{} err := w.UnmarshalConfig(yamlConfig) if err != nil { return errors.Wrap(err, "unable to parse waf configuration") } w.logger = logger - //w.config = wc + w.logger.Logger.SetLevel(*w.config.LogLevel) w.logger.Tracef("WAF configuration: %+v", w.config) @@ -167,27 +159,14 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { wafUUID := uuid.New().String() - var wafLogger *log.Entry - - //configure logger - var clog = log.New() - if err := types.ConfigureLogger(clog); err != nil { - log.Fatalf("While creating bucket-specific logger : %s", err) - } - //is it still needed ? - if w.config.Debug { - clog.SetLevel(log.DebugLevel) - } - wafLogger = clog.WithFields(log.Fields{ - "uuid": wafUUID, - }) - //we copy WaapRutime for each runner wrt := *w.WaapRuntime runner := WaapRunner{ - inChan: w.InChan, - UUID: wafUUID, - logger: wafLogger, + inChan: w.InChan, + UUID: wafUUID, + logger: w.logger.WithFields(log.Fields{ + "uuid": wafUUID, + }), WaapRuntime: &wrt, } err := runner.Init(hub.GetDataDir()) @@ -195,7 +174,6 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { return fmt.Errorf("unable to initialize runner : %s", err) } w.WaapRunners[nbRoutine] = runner - //most likely missign somethign here to actually start the runner :) } w.logger.Infof("Created %d waf runners", len(w.WaapRunners)) From 03650401c5395f0617810878f77906643921b1fc Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 10:57:22 +0200 Subject: [PATCH 118/263] default level --- pkg/acquisition/modules/waap/waap.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 272c50536a1..a8b6fd411a6 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -65,6 +65,11 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { return errors.Wrap(err, "Cannot parse waf configuration") } + if wc.config.LogLevel == nil { + level := new(log.Level) + *level = log.InfoLevel + wc.config.LogLevel = level + } if wc.config.ListenAddr == "" { return fmt.Errorf("listen_addr cannot be empty") } From 685006508c8717e2311938ff8b73e423c933097a Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 13:43:27 +0200 Subject: [PATCH 119/263] make waap rules generate crowdsec events (again) --- pkg/acquisition/modules/waap/utils.go | 46 ++++++++++----------- pkg/acquisition/modules/waap/waap_runner.go | 33 +++++++++------ 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 4bae89b458b..8353239fe95 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -5,7 +5,6 @@ import ( "time" "github.com/crowdsecurity/coraza/v3/collection" - "github.com/crowdsecurity/coraza/v3/experimental" "github.com/crowdsecurity/coraza/v3/types/variables" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" @@ -76,32 +75,23 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { } -/* - how to configure variables to be kept: - 1) full collection : tx.* - 2) subvariables : tx.a* - -*/ - -// func LogWaapEvent(evt *types.Event) error { - -// return nil -// } - -func AccumulateTxToEvent(logger log.Entry, tx experimental.FullTransaction, kind string, evt *types.Event, wr *waf.WaapRuntimeConfig) error { - - if tx.IsInterrupted() { +func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest) error { + if evt == nil { + //an error was already emitted, let's not spam the logs + return nil + } + if req.Tx.IsInterrupted() { if evt.Meta == nil { evt.Meta = map[string]string{} } - if kind == InBand { + if req.IsInBand { evt.Meta["waap_interrupted"] = "true" - evt.Meta["waap_action"] = tx.Interruption().Action + evt.Meta["waap_action"] = req.Tx.Interruption().Action evt.Parsed["inband_interrupted"] = "true" - evt.Parsed["inband_action"] = tx.Interruption().Action + evt.Parsed["inband_action"] = req.Tx.Interruption().Action } else { evt.Parsed["outofband_interrupted"] = "true" - evt.Parsed["outofband_action"] = tx.Interruption().Action + evt.Parsed["outofband_action"] = req.Tx.Interruption().Action } } @@ -109,7 +99,7 @@ func AccumulateTxToEvent(logger log.Entry, tx experimental.FullTransaction, kind evt.Waap.Vars = map[string]string{} } - tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { + req.Tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { for _, variable := range col.FindAll() { key := "" if variable.Key() == "" { @@ -120,23 +110,28 @@ func AccumulateTxToEvent(logger log.Entry, tx experimental.FullTransaction, kind if variable.Value() == "" { continue } - for _, collectionToKeep := range wr.CompiledVariablesTracking { + for _, collectionToKeep := range r.WaapRuntime.CompiledVariablesTracking { match := collectionToKeep.MatchString(key) if match { evt.Waap.Vars[key] = variable.Value() - logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } else { - logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) + r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) } } } return true }) - for _, rule := range tx.MatchedRules() { + for _, rule := range req.Tx.MatchedRules() { if rule.Message() == "" { + r.logger.Tracef("discarding rule %d", rule.Rule().ID()) continue } + kind := "outofband" + if req.IsInBand { + kind = "inband" + } WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() corazaRule := map[string]interface{}{ @@ -158,4 +153,5 @@ func AccumulateTxToEvent(logger log.Entry, tx experimental.FullTransaction, kind } return nil + } diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index ed52abebb66..566eb4607ef 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -165,14 +165,22 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Errorf("unable to process PreEval rules: %s", err) continue } - log.Infof("now response is -> %s", r.WaapRuntime.Response.Action) //inband WAAP rules err = r.ProcessInBandRules(&request) if err != nil { r.logger.Errorf("unable to process InBand rules: %s", err) continue } - + //create the associated event for crowdsec itself + evt, err := EventFromRequest(request) + if err != nil { + //let's not interrupt the pipeline for this + r.logger.Errorf("unable to create event from request : %s", err) + } + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("inband rules matched : %d", in.RuleID) r.WaapRuntime.Response.InBandInterrupt = true @@ -200,20 +208,21 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Errorf("unable to process OutOfBand rules: %s", err) continue } - + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("outband rules matched : %d", in.RuleID) r.WaapRuntime.Response.OutOfBandInterrupt = true - } else { - continue - } - - err = r.WaapRuntime.ProcessOnMatchRules(request) - if err != nil { - r.logger.Errorf("unable to process OnMatch rules: %s", err) - continue + err = r.WaapRuntime.ProcessOnMatchRules(request) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + continue + } } - + r.logger.Debugf("sending event %p to outChan", &evt) + r.outChan <- evt } } } From dd4962092293147a7701532cb9883b59c84533bd Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 17:23:29 +0200 Subject: [PATCH 120/263] our shortcut for waap events --- cmd/crowdsec/parse.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index aa93b6ec7f9..13ae216fc52 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -22,6 +22,13 @@ LOOP: if !event.Process { continue } + /*Waap is going to generate 2 events: + - one that is treated as a log and can go to scenarios + - another one that will go directly to LAPI*/ + if event.Type == types.WAAP { + outputEventChan <- event + continue + } if event.Line.Module == "" { log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line) continue From b2bb15bb494e39997c261ad89487b213836c6c4e Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 17:23:46 +0200 Subject: [PATCH 121/263] generate a special event for waap --- pkg/acquisition/modules/waap/utils.go | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 8353239fe95..007ace68bec 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -6,12 +6,49 @@ import ( "github.com/crowdsecurity/coraza/v3/collection" "github.com/crowdsecurity/coraza/v3/types/variables" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" + "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) +func WaapEventGeneration(inEvt types.Event) (types.Event, error) { + evt := types.Event{} + evt.Type = types.WAAP + evt.Process = true + source := models.Source{ + Value: ptr.Of(inEvt.Parsed["source_ip"]), + IP: inEvt.Parsed["source_ip"], + Scope: ptr.Of(types.Ip), + } + + evt.Overflow.Sources = make(map[string]models.Source) + evt.Overflow.Sources["ip"] = source + + alert := models.Alert{} + alert.Capacity = ptr.Of(int32(1)) + alert.Events = make([]*models.Event, 0) //TBD + alert.Meta = make(models.Meta, 0) //TBD + alert.EventsCount = ptr.Of(int32(1)) + alert.Labels = []string{"waf"} //don't know what to do about this + alert.Leakspeed = ptr.Of("") + msg := fmt.Sprintf("WAF alert: %s", inEvt.Waap.MatchedRules.GetName()) + alert.Message = &msg + alert.Scenario = ptr.Of(inEvt.Waap.MatchedRules.GetName()) + alert.ScenarioHash = ptr.Of(inEvt.Waap.MatchedRules.GetHash()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() + alert.ScenarioVersion = ptr.Of(inEvt.Waap.MatchedRules.GetVersion()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetVersion() + alert.Simulated = ptr.Of(false) + alert.Source = &source + alert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) + alert.StopAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) + + evt.Overflow.APIAlerts = []models.Alert{alert} + evt.Overflow.Alert = &alert + return evt, nil +} + func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? @@ -76,6 +113,7 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { } func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest) error { + if evt == nil { //an error was already emitted, let's not spam the logs return nil From c02c74b5feae66e87246111a7caec150f6292bf0 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 17:24:16 +0200 Subject: [PATCH 122/263] shortcut for waap events --- pkg/acquisition/modules/waap/waap_runner.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 566eb4607ef..6bfca0e1291 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -221,8 +221,15 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { continue } } - r.logger.Debugf("sending event %p to outChan", &evt) r.outChan <- evt + /*we generate a second event that will go directly to LAPI. + we don't want to risk losing all visibility on waap events if the user is missing a scenario*/ + waapOvlfw, err := WaapEventGeneration(evt) + if err != nil { + r.logger.Errorf("unable to generate waap event : %s", err) + } else { + r.outChan <- waapOvlfw + } } } } From 1f3801f390d30941fc32f06fb66615896cd08abb Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 17:24:31 +0200 Subject: [PATCH 123/263] add the helpers and the type --- pkg/types/event.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/types/event.go b/pkg/types/event.go index b60c9a823a4..6631e4bb90b 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -15,6 +15,7 @@ import ( const ( LOG = iota OVFLW + WAAP ) /* @@ -82,6 +83,28 @@ func (w MatchedRules) GetURI() string { return "" } +func (w MatchedRules) GetHash() string { + for _, rule := range w { + return rule["hash"].(string) + } + return "" +} + +func (w MatchedRules) GetVersion() string { + for _, rule := range w { + return rule["version"].(string) + } + return "" +} + +func (w MatchedRules) GetName() string { + for _, rule := range w { + //@sbl : let's fix this, we want to be able to the name of the rule :) + return rule["name"].(string) + } + return "" +} + func (w MatchedRules) GetMethod() string { for _, rule := range w { return rule["method"].(string) From 9edde09608754740111d32224cfcf92197deb834 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 18:16:30 +0200 Subject: [PATCH 124/263] up --- pkg/acquisition/modules/waap/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 007ace68bec..cb7b27e7876 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -29,14 +29,14 @@ func WaapEventGeneration(inEvt types.Event) (types.Event, error) { alert := models.Alert{} alert.Capacity = ptr.Of(int32(1)) - alert.Events = make([]*models.Event, 0) //TBD - alert.Meta = make(models.Meta, 0) //TBD + alert.Events = make([]*models.Event, 0) //@tko -> URI, method, UA, param name + alert.Meta = make(models.Meta, 0) //@tko -> URI, method, UA, param name alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"waf"} //don't know what to do about this alert.Leakspeed = ptr.Of("") msg := fmt.Sprintf("WAF alert: %s", inEvt.Waap.MatchedRules.GetName()) alert.Message = &msg - alert.Scenario = ptr.Of(inEvt.Waap.MatchedRules.GetName()) + alert.Scenario = ptr.Of(inEvt.Waap.MatchedRules.GetName()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() alert.ScenarioHash = ptr.Of(inEvt.Waap.MatchedRules.GetHash()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() alert.ScenarioVersion = ptr.Of(inEvt.Waap.MatchedRules.GetVersion()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetVersion() alert.Simulated = ptr.Of(false) From eafffe7c947fb03e6899bbf1698d7faf4cbf08da Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 24 Oct 2023 18:16:39 +0200 Subject: [PATCH 125/263] up --- pkg/types/event.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/types/event.go b/pkg/types/event.go index 6631e4bb90b..83e5ee76a13 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -85,6 +85,7 @@ func (w MatchedRules) GetURI() string { func (w MatchedRules) GetHash() string { for _, rule := range w { + //@sbl : let's fix this return rule["hash"].(string) } return "" @@ -92,6 +93,7 @@ func (w MatchedRules) GetHash() string { func (w MatchedRules) GetVersion() string { for _, rule := range w { + //@sbl : let's fix this return rule["version"].(string) } return "" @@ -99,7 +101,7 @@ func (w MatchedRules) GetVersion() string { func (w MatchedRules) GetName() string { for _, rule := range w { - //@sbl : let's fix this, we want to be able to the name of the rule :) + //@sbl : let's fix this return rule["name"].(string) } return "" From 4bfca8cab57a36b5ed5580897433e56feb3bdabe Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 25 Oct 2023 13:54:57 +0200 Subject: [PATCH 126/263] fix meta encoding --- pkg/acquisition/modules/waap/utils.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index cb7b27e7876..95b653fa1ce 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -1,6 +1,7 @@ package wafacquisition import ( + "encoding/json" "fmt" "time" @@ -31,6 +32,21 @@ func WaapEventGeneration(inEvt types.Event) (types.Event, error) { alert.Capacity = ptr.Of(int32(1)) alert.Events = make([]*models.Event, 0) //@tko -> URI, method, UA, param name alert.Meta = make(models.Meta, 0) //@tko -> URI, method, UA, param name + for _, key := range []string{"target_uri", "method"} { + + valueByte, err := json.Marshal([]string{inEvt.Parsed[key]}) + if err != nil { + log.Debugf("unable to serialize key %s", key) + continue + } + + meta := models.MetaItems0{ + Key: key, + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + log.Infof("adding Meta - %s = %s", key, inEvt.Parsed[key]) + } alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"waf"} //don't know what to do about this alert.Leakspeed = ptr.Of("") From 676352b5b118d6c95ce4f9563fc387279fcf8586 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 25 Oct 2023 18:45:49 +0200 Subject: [PATCH 127/263] new custom rule format --- pkg/acquisition/modules/waap/utils.go | 3 + pkg/waf/waap_rule.go | 100 ----------------- pkg/waf/waap_rule/modsec_rule_test.go | 67 ++++++++++++ pkg/waf/waap_rule/modsecurity.go | 152 ++++++++++++++++++++++++++ pkg/waf/waap_rule/types.go | 5 + pkg/waf/waap_rule/waap_rule.go | 65 +++++++++++ pkg/waf/waap_rule_test.go | 126 --------------------- pkg/waf/waap_rules_collection.go | 24 ++-- 8 files changed, 307 insertions(+), 235 deletions(-) delete mode 100644 pkg/waf/waap_rule.go create mode 100644 pkg/waf/waap_rule/modsec_rule_test.go create mode 100644 pkg/waf/waap_rule/modsecurity.go create mode 100644 pkg/waf/waap_rule/types.go create mode 100644 pkg/waf/waap_rule/waap_rule.go delete mode 100644 pkg/waf/waap_rule_test.go diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 95b653fa1ce..9b8e1d89235 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -202,6 +202,9 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest "accuracy": rule.Rule().Accuracy(), "msg": rule.Message(), "severity": rule.Rule().Severity().String(), + "name": "FIXFIXFIXFIXFIX", + "hash": "FIXIFIX", + "version": "FIXFIXFIX", } evt.Waap.MatchedRules = append(evt.Waap.MatchedRules, corazaRule) } diff --git a/pkg/waf/waap_rule.go b/pkg/waf/waap_rule.go deleted file mode 100644 index eabee586682..00000000000 --- a/pkg/waf/waap_rule.go +++ /dev/null @@ -1,100 +0,0 @@ -package waf - -import ( - "fmt" - "strings" - "time" -) - -type VPatchRule struct { - //Those 2 together represent something like ARGS.foo - //If only target is set, it's used for variables that are not a collection (REQUEST_METHOD, etc) - Target string `yaml:"target"` - Variable string `yaml:"var"` - - Match string `yaml:"match"` //@rx - Equals string `yaml:"equals"` //@eq - Transform string `yaml:"transform"` //t:lowercase, t:uppercase, etc - Detect string `yaml:"detect"` //@detectXSS, @detectSQLi, etc - Logic string `yaml:"logic,omitempty"` // "AND", "OR", or empty if not applicable - SubRules []VPatchRule `yaml:"sub_rules,omitempty"` -} - -func (v *VPatchRule) String() string { - return strings.Trim(v.constructRule(0), "\n") -} - -func countTotalRules(rules []VPatchRule) int { - count := 0 - for _, rule := range rules { - count++ - if rule.Logic == "AND" { - count += countTotalRules(rule.SubRules) - } - } - return count -} - -func (v *VPatchRule) constructRule(depth int) string { - var result string - result = v.singleRuleString() - - if len(v.SubRules) == 0 { - return result + "\n" - } - - switch v.Logic { - case "AND": - result = strings.TrimSuffix(result, `"`) + `,chain"` + "\n" - for _, subRule := range v.SubRules { - result += subRule.constructRule(depth + 1) - } - case "OR": - skips := countTotalRules(v.SubRules) - 1 - if depth == 0 { - skips++ - } - result = strings.TrimSuffix(result, `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) - for _, subRule := range v.SubRules { - skips-- - if skips > 0 { - result += strings.TrimSuffix(subRule.singleRuleString(), `"`) + fmt.Sprintf(`,skip:%d"`+"\n", skips) - } else { - result += subRule.singleRuleString() + "\n" - } - } - } - return result -} - -func (v *VPatchRule) singleRuleString() string { - var operator string - var ruleStr string - - if v.Match != "" { - operator = fmt.Sprintf("@rx %s", v.Match) - } else if v.Equals != "" { - operator = fmt.Sprintf("@eq %s", v.Equals) - } else { - return "" - } - - if v.Variable != "" { - ruleStr = fmt.Sprintf(`SecRule %s:%s "%s"`, v.Target, v.Variable, operator) - } else { - ruleStr = fmt.Sprintf(`SecRule %s "%s"`, v.Target, operator) - } - - //FIXME: phase2 should probably not be hardcoded - //Find a better way than using time.Now().UnixMilli() to generate a unique ID - actions := fmt.Sprintf(` "id:%d,deny,log,phase:2`, time.Now().UnixNano()) - - // Handle transformation - if v.Transform != "" { - actions = actions + fmt.Sprintf(",t:%s", v.Transform) - } - actions = actions + `"` - ruleStr = ruleStr + actions - - return ruleStr -} diff --git a/pkg/waf/waap_rule/modsec_rule_test.go b/pkg/waf/waap_rule/modsec_rule_test.go new file mode 100644 index 00000000000..3f40eff6949 --- /dev/null +++ b/pkg/waf/waap_rule/modsec_rule_test.go @@ -0,0 +1,67 @@ +package waap_rule + +import "testing" + +func TestVPatchRuleString(t *testing.T) { + tests := []struct { + name string + rule CustomRule + expected string + }{ + { + name: "Base Rule", + rule: CustomRule{ + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1136235475,phase:2,deny,log,msg:'Base Rule',t:lowercase"`, + }, + { + name: "Multiple Zones", + rule: CustomRule{ + Zones: []string{"ARGS", "BODY_ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + expected: `SecRule ARGS_GET:foo|ARGS_POST:foo "@rx [^a-zA-Z]" "id:2088895799,phase:2,deny,log,msg:'Multiple Zones',t:lowercase"`, + }, + { + name: "Basic AND", + rule: CustomRule{ + And: []CustomRule{ + { + + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2323451654,phase:2,deny,log,msg:'Basic AND_and_0',t:lowercase,chain" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2075918819,phase:2,deny,log,msg:'Basic AND_and_1',t:lowercase"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := tt.rule.Convert(ModsecurityRuleType, tt.name) + + if err != nil { + t.Errorf("Error converting rule: %s", err) + } + if actual != tt.expected { + t.Errorf("Expected:\n%s\nGot:\n%s", tt.expected, actual) + } + }) + } +} diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go new file mode 100644 index 00000000000..438cea2eff9 --- /dev/null +++ b/pkg/waf/waap_rule/modsecurity.go @@ -0,0 +1,152 @@ +package waap_rule + +import ( + "fmt" + "hash/fnv" + "strings" +) + +type ModsecurityRule struct { +} + +var zonesMap map[string]string = map[string]string{ + "ARGS": "ARGS_GET", + "ARGS_NAMES": "ARGS_GET_NAMES", + "BODY_ARGS": "ARGS_POST", + "BODY_ARGS_NAMES": "ARGS_POST_NAMES", + "HEADERS": "REQUEST_HEADERS", + "METHOD": "REQUEST_METHOD", + "PROTOCOL": "REQUEST_PROTOCOL", + "URI": "REQUEST_URI", +} + +var transformMap map[string]string = map[string]string{ + "lowercase": "t:lowercase", + "uppercase": "t:uppercase", + "b64decode": "t:base64Decode", + "hexdecode": "t:hexDecode", + "length": "t:length", +} + +var matchMap map[string]string = map[string]string{ + "regex": "@rx", + "equal": "@streq", + "startsWith": "@beginsWith", + "endsWith": "@endsWith", + "contains": "@contains", + "libinjectionSQL": "@detectSQLi", + "libinjectionXSS": "@detectXSS", + "gt": "@gt", + "lt": "@lt", + "ge": "@ge", + "le": "@le", +} + +func (m ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, error) { + + rules, err := m.buildRules(rule, waapRuleName, false) + + if err != nil { + return "", err + } + + return strings.Join(rules, "\n"), nil +} + +func (m ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) uint32 { + h := fnv.New32a() + h.Write([]byte(waapRuleName)) + h.Write([]byte(rule.Match.Type)) + h.Write([]byte(rule.Match.Value)) + for _, zone := range rule.Zones { + h.Write([]byte(zone)) + } + for _, transform := range rule.Transform { + h.Write([]byte(transform)) + } + return h.Sum32() +} + +func (m ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool) ([]string, error) { + ret := make([]string, 0) + + if rule.And != nil { + for c, andRule := range rule.And { + subName := fmt.Sprintf("%s_and_%d", waapRuleName, c) + lastRule := c == len(rule.And)-1 + rules, err := m.buildRules(&andRule, subName, !lastRule) + if err != nil { + return nil, err + } + ret = append(ret, rules...) + } + } + + if rule.Or != nil { + for c, orRule := range rule.Or { + subName := fmt.Sprintf("%s_or_%d", waapRuleName, c) + rules, err := m.buildRules(&orRule, subName, false) + if err != nil { + return nil, err + } + ret = append(ret, rules...) + } + } + + r := strings.Builder{} + + r.WriteString("SecRule ") + + if rule.Zones == nil { + return ret, nil + } + + for idx, zone := range rule.Zones { + mappedZone, ok := zonesMap[zone] + if !ok { + return nil, fmt.Errorf("unknown zone '%s'", zone) + } + if len(rule.Variables) == 0 { + r.WriteString(mappedZone) + } else { + for j, variable := range rule.Variables { + if idx > 0 || j > 0 { + r.WriteByte('|') + } + r.WriteString(fmt.Sprintf("%s:%s", mappedZone, variable)) + } + } + } + r.WriteByte(' ') + + if rule.Match.Type != "" { + if match, ok := matchMap[rule.Match.Type]; ok { + r.WriteString(fmt.Sprintf(`"%s %s"`, match, rule.Match.Value)) + } else { + return nil, fmt.Errorf("unknown match type '%s'", rule.Match.Type) + } + } + + //Should phase:2 be configurable? + r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s'`, m.generateRuleID(rule, waapRuleName), waapRuleName)) + + if rule.Transform != nil { + for _, transform := range rule.Transform { + r.WriteByte(',') + if mappedTransform, ok := transformMap[transform]; ok { + r.WriteString(mappedTransform) + } else { + return nil, fmt.Errorf("unknown transform '%s'", transform) + } + } + } + + if and { + r.WriteString(",chain") + } + + r.WriteByte('"') + + ret = append(ret, r.String()) + return ret, nil +} diff --git a/pkg/waf/waap_rule/types.go b/pkg/waf/waap_rule/types.go new file mode 100644 index 00000000000..80ac560a0fd --- /dev/null +++ b/pkg/waf/waap_rule/types.go @@ -0,0 +1,5 @@ +package waap_rule + +const ( + ModsecurityRuleType = "modsecurity" +) diff --git a/pkg/waf/waap_rule/waap_rule.go b/pkg/waf/waap_rule/waap_rule.go new file mode 100644 index 00000000000..00d3210d18e --- /dev/null +++ b/pkg/waf/waap_rule/waap_rule.go @@ -0,0 +1,65 @@ +package waap_rule + +import ( + "fmt" +) + +/* +rules: + - name: "test" + and: + - zones: + - BODY_ARGS + variables: + - foo + - bar + transform: + - lowercase|uppercase|b64decode|... + match: + type: regex + value: "[^a-zA-Z]" + - zones: + - ARGS + variables: + - bla + +*/ + +type match struct { + Type string `yaml:"type"` + Value string `yaml:"value"` +} + +type CustomRule struct { + Name string `yaml:"name"` + + Zones []string `yaml:"zones"` + Variables []string `yaml:"variables"` + + Match match `yaml:"match"` + Transform []string `yaml:"transform"` //t:lowercase, t:uppercase, etc + And []CustomRule `yaml:"and,omitempty"` + Or []CustomRule `yaml:"or,omitempty"` +} + +func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, error) { + + if v.Zones == nil && v.And == nil && v.Or == nil { + return "", fmt.Errorf("no zones defined") + } + + if v.Match.Type == "" && v.And == nil && v.Or == nil { + return "", fmt.Errorf("no match type defined") + } + + if v.Match.Value == "" && v.And == nil && v.Or == nil { + return "", fmt.Errorf("no match value defined") + } + + switch ruleType { + case ModsecurityRuleType: + return ModsecurityRule{}.Build(v, waapRuleName) + default: + return "", fmt.Errorf("unknown rule format '%s'", ruleType) + } +} diff --git a/pkg/waf/waap_rule_test.go b/pkg/waf/waap_rule_test.go deleted file mode 100644 index fcb45a2d39a..00000000000 --- a/pkg/waf/waap_rule_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package waf - -import "testing" - -func TestVPatchRuleString(t *testing.T) { - tests := []struct { - name string - rule VPatchRule - expected string - }{ - { - name: "Base Rule", - rule: VPatchRule{ - Target: "ARGS", - Variable: "foo", - Match: "[^a-zA-Z]", - Transform: "lowercase", - }, - expected: `SecRule ARGS:foo "@rx [^a-zA-Z]" "id:0,deny,log,t:lowercase"`, - }, - { - name: "AND Logic Rule", - rule: VPatchRule{ - Target: "ARGS", - Variable: "bar", - Match: "[0-9]", - Logic: "AND", - SubRules: []VPatchRule{ - { - Target: "REQUEST_URI", - Match: "/joomla/index.php/component/users/", - }, - }, - }, - expected: `SecRule ARGS:bar "@rx [0-9]" "id:0,deny,log,chain" -SecRule REQUEST_URI "@rx /joomla/index.php/component/users/" "id:0,deny,log"`, - }, - { - name: "AND Logic Rule", - rule: VPatchRule{ - Logic: "AND", - SubRules: []VPatchRule{ - { - Target: "REQUEST_URI", - Match: "/joomla/index.php/component/users/", - }, - { - Target: "ARGS", - Variable: "bar", - Match: "[0-9]", - }, - }, - }, - expected: `SecRule ARGS:bar "@rx [0-9]" "id:0,deny,log,chain" -SecRule REQUEST_URI "@rx /joomla/index.php/component/users/" "id:0,deny,log"`, - }, - { - name: "OR Logic Rule", - rule: VPatchRule{ - Target: "REQUEST_HEADERS", - Variable: "User-Agent", - Match: "BadBot", - Logic: "OR", - SubRules: []VPatchRule{ - { - Target: "REQUEST_HEADERS", - Variable: "Referer", - Match: "EvilReferer", - }, - { - Target: "REQUEST_METHOD", - Equals: "POST", - }, - }, - }, - expected: `SecRule REQUEST_HEADERS:User-Agent "@rx BadBot" "id:0,deny,log,skip:2" -SecRule REQUEST_HEADERS:Referer "@rx EvilReferer" "id:0,deny,log,skip:1" -SecRule REQUEST_METHOD "@eq POST" "id:0,deny,log"`, - }, - { - name: "AND-OR Logic Mix", - rule: VPatchRule{ - Target: "REQUEST_URI", - Match: "/api/", - Logic: "AND", - SubRules: []VPatchRule{ - { - Target: "ARGS", - Variable: "username", - Match: "admin", - Logic: "OR", - SubRules: []VPatchRule{ - { - Target: "REQUEST_METHOD", - Equals: "POST", - Logic: "AND", - SubRules: []VPatchRule{ - { - Target: "ARGS", - Variable: "action", - Match: "delete", - }, - }, - }, - }, - }, - }, - }, - expected: `SecRule REQUEST_URI "@rx /api/" "id:0,deny,log,chain" -SecRule ARGS:username "@rx admin" "id:0,deny,log,skip:2" -SecRule REQUEST_METHOD "@eq POST" "id:0,deny,log,chain" -SecRule ARGS:action "@rx delete" "id:0,deny,log"`, - }, - // Additional OR test case would be here, but note that the OR logic representation with `skip` is very simplistic. - // It may not be robust enough for complex OR rules in a real-world ModSecurity setup. - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actual := tt.rule.String() - if actual != tt.expected { - t.Errorf("Expected:\n%s\nGot:\n%s", tt.expected, actual) - } - }) - } -} diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 6d05245db60..02cd12e87f3 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -8,6 +8,7 @@ import ( corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" "gopkg.in/yaml.v2" log "github.com/sirupsen/logrus" @@ -23,13 +24,13 @@ var WAAP_RULE = "waap-rule" // to be filled w/ seb update type WaapCollectionConfig struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - Description string `yaml:"description"` - SecLangFilesRules []string `yaml:"seclang_files_rules"` - SecLangRules []string `yaml:"seclang_rules"` - Rules []VPatchRule `yaml:"rules"` - Data interface{} `yaml:"data"` //Ignore it + Type string `yaml:"type"` + Name string `yaml:"name"` + Description string `yaml:"description"` + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + Rules []waap_rule.CustomRule `yaml:"rules"` + Data interface{} `yaml:"data"` //Ignore it } func LoadCollection(collection string) (WaapCollection, error) { @@ -115,8 +116,13 @@ func LoadCollection(collection string) (WaapCollection, error) { if loadedRule.Rules != nil { for _, rule := range loadedRule.Rules { - log.Infof("Adding rule %s", rule.String()) - waapCol.Rules = append(waapCol.Rules, rule.String()) + strRule, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) + if err != nil { + log.Errorf("unable to convert rule %s : %s", rule.Name, err) + return WaapCollection{}, err + } + log.Infof("Adding rule %s", strRule) + waapCol.Rules = append(waapCol.Rules, strRule) } } From 46ae0b382218dc9d5889cd811974cd766f537cbd Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 12:03:57 +0200 Subject: [PATCH 128/263] properly set default log level --- pkg/waf/waap.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 4982c5763d2..997f1799bd2 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -81,7 +81,7 @@ type WaapConfig struct { PreEval []Hook `yaml:"pre_eval"` OnMatch []Hook `yaml:"on_match"` VariablesTracking []string `yaml:"variables_tracking"` - LogLevel log.Level `yaml:"log_level"` + LogLevel *log.Level `yaml:"log_level"` Logger *log.Entry `yaml:"-"` } @@ -110,8 +110,12 @@ func (wc *WaapConfig) Load(file string) error { if wc.Name == "" { return fmt.Errorf("name cannot be empty") } + if wc.LogLevel == nil { + lvl := log.InfoLevel + wc.LogLevel = &lvl + } wc.Logger = wc.Logger.WithField("name", wc.Name) - wc.Logger.Logger.SetLevel(wc.LogLevel) + wc.Logger.Logger.SetLevel(*wc.LogLevel) if wc.DefaultRemediation == "" { return fmt.Errorf("default_remediation cannot be empty") } From 6cbeefead69a05713993a79712afa0b44067539c Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 12:04:58 +0200 Subject: [PATCH 129/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 6bfca0e1291..db7a8fcda86 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -42,7 +42,6 @@ func (r *WaapRunner) Init(datadir string) error { } //adapt the logger level to the WAAP runnerLogger := r.logger.Dup() - runnerLogger.Logger.SetLevel(r.WaapRuntime.Config.LogLevel) runnerLogger.Infof("setting logger of %s to %s", r.WaapRuntime.Name, r.WaapRuntime.Config.LogLevel) r.WaapInbandEngine, err = coraza.NewWAF( coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), From f18b5541771440ee28703d50af167f4c56d71cd3 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 12:45:59 +0200 Subject: [PATCH 130/263] warn at start if body reading is disabled --- pkg/acquisition/modules/waap/waap_runner.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index db7a8fcda86..53bfbfc380e 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -51,6 +51,11 @@ func (r *WaapRunner) Init(datadir string) error { return fmt.Errorf("unable to initialize inband engine : %w", err) } + tx := r.WaapInbandEngine.NewTransaction() + if !tx.IsRequestBodyAccessible() { + runnerLogger.Warningf("request body is not accessible, inband rules won't be able to match on it") + } + r.WaapOutbandEngine, err = coraza.NewWAF( coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), ) @@ -59,6 +64,11 @@ func (r *WaapRunner) Init(datadir string) error { return fmt.Errorf("unable to initialize outband engine : %w", err) } + tx = r.WaapOutbandEngine.NewTransaction() + if !tx.IsRequestBodyAccessible() { + runnerLogger.Warningf("request body is not accessible, outband rules won't be able to match on it") + } + return nil } From 82bb8a2789b64b5f83ea37a5bd389275c1419696 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 13:01:11 +0200 Subject: [PATCH 131/263] no leak plz --- pkg/acquisition/modules/waap/waap_runner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 53bfbfc380e..85b45220f17 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -55,6 +55,7 @@ func (r *WaapRunner) Init(datadir string) error { if !tx.IsRequestBodyAccessible() { runnerLogger.Warningf("request body is not accessible, inband rules won't be able to match on it") } + tx.Close() r.WaapOutbandEngine, err = coraza.NewWAF( coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), @@ -68,6 +69,7 @@ func (r *WaapRunner) Init(datadir string) error { if !tx.IsRequestBodyAccessible() { runnerLogger.Warningf("request body is not accessible, outband rules won't be able to match on it") } + tx.Close() return nil } From 0cebf833c753c8d6e1d5d37220cc3982f64d35e6 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 14:46:08 +0200 Subject: [PATCH 132/263] add options via WaapConfig for inband and outofband engines --- pkg/acquisition/modules/waap/waap_runner.go | 40 +++++++++++---------- pkg/waf/waap.go | 40 +++++++++++++-------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 85b45220f17..9527eece288 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -40,37 +40,39 @@ func (r *WaapRunner) Init(datadir string) error { for _, collection := range r.WaapRuntime.OutOfBandRules { outOfBandRules += collection.String() } - //adapt the logger level to the WAAP runnerLogger := r.logger.Dup() - runnerLogger.Infof("setting logger of %s to %s", r.WaapRuntime.Name, r.WaapRuntime.Config.LogLevel) - r.WaapInbandEngine, err = coraza.NewWAF( - coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), - ) + //setting up inband engine + inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)) + if !r.WaapRuntime.Config.InbandOptions.DisableBodyInspection { + inbandCfg = inbandCfg.WithRequestBodyAccess() + } else { + log.Warningf("Disabling body inspection, Inband rules will not be able to match on body's content.") + } + if r.WaapRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil { + inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.WaapRuntime.Config.InbandOptions.RequestBodyInMemoryLimit) + } + r.WaapInbandEngine, err = coraza.NewWAF(inbandCfg) if err != nil { return fmt.Errorf("unable to initialize inband engine : %w", err) } - tx := r.WaapInbandEngine.NewTransaction() - if !tx.IsRequestBodyAccessible() { - runnerLogger.Warningf("request body is not accessible, inband rules won't be able to match on it") + //setting up outband engine + outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)) + if !r.WaapRuntime.Config.OutOfBandOptions.DisableBodyInspection { + outbandCfg = outbandCfg.WithRequestBodyAccess() + } else { + log.Warningf("Disabling body inspection, Out of band rules will not be able to match on body's content.") } - tx.Close() - - r.WaapOutbandEngine, err = coraza.NewWAF( - coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)), - ) + if r.WaapRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil { + outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.WaapRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit) + } + r.WaapOutbandEngine, err = coraza.NewWAF(outbandCfg) if err != nil { return fmt.Errorf("unable to initialize outband engine : %w", err) } - tx = r.WaapOutbandEngine.NewTransaction() - if !tx.IsRequestBodyAccessible() { - runnerLogger.Warningf("request body is not accessible, outband rules won't be able to match on it") - } - tx.Close() - return nil } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 997f1799bd2..9846b7c222b 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -48,12 +48,18 @@ type WaapTempResponse struct { SendEvent bool //do we send an internal event on rule match } +type WaapSubEngineOpts struct { + DisableBodyInspection bool `yaml:"disable_body_inspection"` + RequestBodyInMemoryLimit *int `yaml:"request_body_in_memory_limit"` +} + // runtime version of WaapConfig type WaapRuntimeConfig struct { Name string OutOfBandRules []WaapCollection - //OutOfBandEngine XXX - InBandRules []WaapCollection + + InBandRules []WaapCollection + DefaultRemediation string CompiledOnLoad []Hook CompiledPreEval []Hook @@ -67,22 +73,26 @@ type WaapRuntimeConfig struct { InBandTx ExtendedTransaction //is it a good idea ? Response WaapTempResponse //should we store matched rules here ? + } type WaapConfig struct { - Name string `yaml:"name"` - OutOfBandRules []string `yaml:"outofband_rules"` - InBandRules []string `yaml:"inband_rules"` - DefaultRemediation string `yaml:"default_remediation"` - DefaultPassAction string `yaml:"default_pass_action"` - BlockedHTTPCode int `yaml:"blocked_http_code"` - PassedHTTPCode int `yaml:"passed_http_code"` - OnLoad []Hook `yaml:"on_load"` - PreEval []Hook `yaml:"pre_eval"` - OnMatch []Hook `yaml:"on_match"` - VariablesTracking []string `yaml:"variables_tracking"` - LogLevel *log.Level `yaml:"log_level"` - Logger *log.Entry `yaml:"-"` + Name string `yaml:"name"` + OutOfBandRules []string `yaml:"outofband_rules"` + InBandRules []string `yaml:"inband_rules"` + DefaultRemediation string `yaml:"default_remediation"` + DefaultPassAction string `yaml:"default_pass_action"` + BlockedHTTPCode int `yaml:"blocked_http_code"` + PassedHTTPCode int `yaml:"passed_http_code"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + OnMatch []Hook `yaml:"on_match"` + VariablesTracking []string `yaml:"variables_tracking"` + InbandOptions WaapSubEngineOpts `yaml:"inband_options"` + OutOfBandOptions WaapSubEngineOpts `yaml:"outofband_options"` + + LogLevel *log.Level `yaml:"log_level"` + Logger *log.Entry `yaml:"-"` } func (w *WaapRuntimeConfig) ClearResponse() { From cd1cefbc8b37168bb6e7f6b6f9dd99eb53272d08 Mon Sep 17 00:00:00 2001 From: bui Date: Thu, 26 Oct 2023 15:23:45 +0200 Subject: [PATCH 133/263] fix behavior so we only generate crowdsec events if interrupt was generated in either inband or outofband phases --- pkg/acquisition/modules/waap/utils.go | 33 ++++++++++++--------- pkg/acquisition/modules/waap/waap_runner.go | 9 ++++-- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 9b8e1d89235..f33ecc341e5 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -72,7 +72,6 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt.ExpectMode = types.LIVE //def needs fixing evt.Stage = "s00-raw" - evt.Process = true evt.Parsed = map[string]string{ "source_ip": r.ClientIP, "target_host": r.Host, @@ -134,19 +133,25 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest //an error was already emitted, let's not spam the logs return nil } - if req.Tx.IsInterrupted() { - if evt.Meta == nil { - evt.Meta = map[string]string{} - } - if req.IsInBand { - evt.Meta["waap_interrupted"] = "true" - evt.Meta["waap_action"] = req.Tx.Interruption().Action - evt.Parsed["inband_interrupted"] = "true" - evt.Parsed["inband_action"] = req.Tx.Interruption().Action - } else { - evt.Parsed["outofband_interrupted"] = "true" - evt.Parsed["outofband_action"] = req.Tx.Interruption().Action - } + + if !req.Tx.IsInterrupted() { + //if the phase didn't generate an interruption, we don't have anything to add to the event + return nil + } + //if one interruption was generated, event is good for processing :) + evt.Process = true + + if evt.Meta == nil { + evt.Meta = map[string]string{} + } + if req.IsInBand { + evt.Meta["waap_interrupted"] = "true" + evt.Meta["waap_action"] = req.Tx.Interruption().Action + evt.Parsed["inband_interrupted"] = "true" + evt.Parsed["inband_action"] = req.Tx.Interruption().Action + } else { + evt.Parsed["outofband_interrupted"] = "true" + evt.Parsed["outofband_action"] = req.Tx.Interruption().Action } if evt.Waap.Vars == nil { diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 9527eece288..a986f5f811e 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -234,9 +234,14 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { continue } } + + if !evt.Process { + continue + } + + //we generate two events: one that is going to be picked up by the acquisition pipeline (parsers, scenarios etc.) + //and a second one that will go straight to LAPI r.outChan <- evt - /*we generate a second event that will go directly to LAPI. - we don't want to risk losing all visibility on waap events if the user is missing a scenario*/ waapOvlfw, err := WaapEventGeneration(evt) if err != nil { r.logger.Errorf("unable to generate waap event : %s", err) From 6b8ed0c9d06b35d924bb99c6c41ff6655b930cbb Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 27 Oct 2023 10:25:29 +0200 Subject: [PATCH 134/263] Refactor hub URL/branch configuration (#2559) * Refactor hub URL/branch configuration * docker: using --force to implement $DISABLE (required for items in collections) * use pointer receiver for consistency --- {pkg/cwhub => cmd/crowdsec-cli}/branch.go | 15 +------- cmd/crowdsec-cli/config_restore.go | 8 ++--- cmd/crowdsec-cli/hub.go | 44 +++++++++++++++-------- cmd/crowdsec-cli/itemcommands.go | 10 ++++-- cmd/crowdsec-cli/main.go | 10 +++--- cmd/crowdsec-cli/require/require.go | 4 +-- cmd/crowdsec-cli/setup.go | 6 ++-- docker/docker_start.sh | 8 ++--- pkg/csconfig/hub.go | 5 +-- pkg/cwhub/cwhub.go | 7 +--- pkg/cwhub/cwhub_test.go | 13 ++++--- pkg/cwhub/enable_test.go | 4 +-- pkg/cwhub/helpers.go | 23 ++++++------ pkg/cwhub/helpers_test.go | 18 +++++----- pkg/cwhub/hub.go | 23 ++++++------ pkg/cwhub/hub_test.go | 17 +++------ pkg/cwhub/sync.go | 8 ++++- pkg/setup/install.go | 12 +++---- 18 files changed, 117 insertions(+), 118 deletions(-) rename {pkg/cwhub => cmd/crowdsec-cli}/branch.go (79%) diff --git a/pkg/cwhub/branch.go b/cmd/crowdsec-cli/branch.go similarity index 79% rename from pkg/cwhub/branch.go rename to cmd/crowdsec-cli/branch.go index e25a6becf31..f4dae2947fe 100644 --- a/pkg/cwhub/branch.go +++ b/cmd/crowdsec-cli/branch.go @@ -1,4 +1,4 @@ -package cwhub +package main // Set the appropriate hub branch according to config settings and crowdsec version @@ -44,16 +44,3 @@ func chooseHubBranch() string { return csVersion } - -// SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() { - // a branch is already set, or specified from the flags - if HubBranch != "" { - return - } - - // use the branch corresponding to the crowdsec version - HubBranch = chooseHubBranch() - - log.Debugf("Using branch '%s' for the hub", HubBranch) -} diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 1a081d6c5ce..d2b31c1d1c7 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -22,7 +22,7 @@ type OldAPICfg struct { } // it's a rip of the cli version, but in silent-mode -func silentInstallItem(name string, obtype string) (string, error) { +func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, error) { hub, err := cwhub.GetHub() if err != nil { return "", err @@ -32,7 +32,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if item == nil { return "", fmt.Errorf("error retrieving item") } - err = hub.DownloadLatest(item, false, false) + err = hub.DownloadLatest(item, false, false, hubURLTemplate, branch) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } @@ -53,7 +53,7 @@ func silentInstallItem(name string, obtype string) (string, error) { func restoreHub(dirPath string) error { var err error - cwhub.SetHubBranch() + branch := chooseHubBranch() for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) @@ -73,7 +73,7 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - label, err := silentInstallItem(toinstall, itype) + label, err := silentInstallItem(toinstall, itype, hubURLTemplate, branch) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) } else if label != "" { diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c0f69ea870f..6023f0e86ef 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -12,6 +12,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) +const ( + hubURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" + remoteIndexPath = ".index.json" +) + +var hubBranch = "" + func NewHubCmd() *cobra.Command { var cmdHub = &cobra.Command{ Use: "hub [action]", @@ -33,7 +40,6 @@ cscli hub upgrade`, return nil }, } - cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub") cmdHub.AddCommand(NewHubListCmd()) cmdHub.AddCommand(NewHubUpdateCmd()) @@ -81,7 +87,7 @@ func NewHubListCmd() *cobra.Command { Short: "List all installed configurations", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: runHubList, + RunE: runHubList, } flags := cmdHubList.Flags() @@ -91,18 +97,23 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - cwhub.SetHubBranch() - // don't use require.Hub because if there is no index file, it would fail - hub, err := cwhub.InitHubUpdate(csConfig.Hub) + branch := hubBranch + if branch == "" { + branch = chooseHubBranch() + } + + log.Debugf("Using branch '%s' for the hub", branch) + + hub, err := cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath) if err != nil { if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) } - log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) - cwhub.HubBranch = "master" - if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil { + log.Warnf("Could not find index file for branch '%s', using 'master'", branch) + branch = "master" + if hub, err = cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } @@ -146,28 +157,35 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { return err } + branch := hubBranch + if branch == "" { + branch = chooseHubBranch() + } + + log.Debugf("Using branch '%s' for the hub", branch) + hub, err := require.Hub(csConfig) if err != nil { return err } log.Infof("Upgrading collections") - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading parsers") - if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading scenarios") - if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading postoverflows") - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force, hubURLTemplate, branch); err != nil { return err } @@ -188,8 +206,6 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - return nil }, RunE: runHubUpgrade, diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 0afa27787eb..df612ab1a89 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -216,6 +216,8 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } + branch := chooseHubBranch() + for _, name := range args { t := hub.GetItem(it.name, name) if t == nil { @@ -225,7 +227,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) continue } - if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, it.name, force, downloadOnly, hubURLTemplate, branch); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -367,8 +369,10 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } + branch := chooseHubBranch() + if all { - if err := hub.UpgradeConfig(it.name, "", force); err != nil { + if err := hub.UpgradeConfig(it.name, "", force, hubURLTemplate, branch); err != nil { return err } return nil @@ -379,7 +383,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) } for _, name := range args { - if err := hub.UpgradeConfig(it.name, name, force); err != nil { + if err := hub.UpgradeConfig(it.name, name, force, hubURLTemplate, branch); err != nil { return err } } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 1d3aa56a656..8f5561d3d2c 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "github.com/fatih/color" @@ -12,9 +11,9 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" + "slices" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -66,9 +65,10 @@ func initConfig() { log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) } - if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" { - cwhub.HubBranch = csConfig.Cscli.HubBranch + if hubBranch == "" && csConfig.Cscli.HubBranch != "" { + hubBranch = csConfig.Cscli.HubBranch } + if OutputFormat != "" { csConfig.Cscli.Output = OutputFormat if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" { @@ -197,7 +197,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error") rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace") - rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github") + rootCmd.PersistentFlags().StringVar(&hubBranch, "branch", "", "Override hub branch on github") if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { log.Fatalf("failed to hide flag: %s", err) } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index b477e9e00c9..2985a06cf2b 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -64,13 +64,11 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub (c *csconfig.Config) (*cwhub.Hub, error) { +func Hub(c *csconfig.Config) (*cwhub.Hub, error) { if c.Hub == nil { return nil, fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - hub, err := cwhub.InitHub(c.Hub) if err != nil { return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go index cc0a9a35d71..f589951e0ec 100644 --- a/cmd/crowdsec-cli/setup.go +++ b/cmd/crowdsec-cli/setup.go @@ -6,10 +6,10 @@ import ( "os" "os/exec" + goccyyaml "github.com/goccy/go-yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - goccyyaml "github.com/goccy/go-yaml" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -298,12 +298,14 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error { return err } + branch := chooseHubBranch() + input, err := os.ReadFile(fromFile) if err != nil { return fmt.Errorf("while reading file %s: %w", fromFile, err) } - if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { + if err = setup.InstallHubItems(csConfig, input, dryRun, hubURLTemplate, branch); err != nil { return err } diff --git a/docker/docker_start.sh b/docker/docker_start.sh index d328cff1af8..420b7b8d9f9 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -327,22 +327,22 @@ fi ## Remove collections, parsers, scenarios & postoverflows if [ "$DISABLE_COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean collections remove "$DISABLE_COLLECTIONS" + cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force fi if [ "$DISABLE_PARSERS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean parsers remove "$DISABLE_PARSERS" + cscli_if_clean parsers remove "$DISABLE_PARSERS" --force fi if [ "$DISABLE_SCENARIOS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" + cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force fi if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" + cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force fi ## Register bouncers via env diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index db49c0231df..d3638c8885d 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,6 +1,6 @@ package csconfig -// HubConfig holds the configuration for a hub +// HubCfg holds the configuration for a hub type HubCfg struct { HubIndexFile string // Path to the local index file HubDir string // Where the hub items are downloaded @@ -9,9 +9,6 @@ type HubCfg struct { } func (c *Config) loadHub() error { - - // XXX: HubBranch too -- from cscli or chooseHubBranch() ? - c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index d1f155813a8..24c2d859eca 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -8,9 +8,4 @@ import ( "errors" ) -var ( - ErrMissingReference = errors.New("Reference(s) missing in collection") - - RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" - HubBranch = "master" -) +var ErrMissingReference = errors.New("Reference(s) missing in collection") diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index b51c3aa6ec4..45e2852bd61 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -14,6 +14,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) +const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" + /* To test : - Download 'first' hub index @@ -58,15 +60,16 @@ func testHub(t *testing.T, update bool) *Hub { os.RemoveAll(tmpDir) }) - constructor := InitHub + var hub *Hub if update { - constructor = InitHubUpdate + hub, err = InitHubUpdate(hubCfg, mockURLTemplate, "master", ".index.json") + require.NoError(t, err) + } else { + hub, err = InitHub(hubCfg) + require.NoError(t, err) } - hub, err := constructor(hubCfg) - require.NoError(t, err) - return hub } diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 864b3fa8587..a5d731fee7b 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -10,7 +10,7 @@ import ( func testInstall(hub *Hub, t *testing.T, item Item) { // Install the parser - err := hub.DownloadLatest(&item, false, false) + err := hub.DownloadLatest(&item, false, false, mockURLTemplate, "master") require.NoError(t, err, "failed to download %s", item.Name) _, err = hub.LocalSync() @@ -51,7 +51,7 @@ func testUpdate(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := hub.DownloadLatest(&item, true, true) + err := hub.DownloadLatest(&item, true, true, mockURLTemplate, "master") require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index ad66336be41..6f42bed65e3 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -19,7 +19,7 @@ import ( ) // InstallItem installs an item from the hub -func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { +func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool, hubURLTemplate, branch string) error { item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) @@ -33,7 +33,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } } - err := h.DownloadLatest(item, force, true) + err := h.DownloadLatest(item, force, true, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } @@ -111,7 +111,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for } // UpgradeConfig upgrades an item from the hub -func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { +func (h *Hub) UpgradeConfig(itemType string, name string, force bool, hubURLTemplate, branch string) error { updated := 0 found := false @@ -144,7 +144,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { } } - if err := h.DownloadLatest(&v, force, true); err != nil { + if err := h.DownloadLatest(&v, force, true, hubURLTemplate, branch); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -185,7 +185,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { } // DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubURLTemplate, branch string) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -196,7 +196,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro return nil } - return h.DownloadItem(target, overwrite) + return h.DownloadItem(target, overwrite, hubURLTemplate, branch) } // collection @@ -216,7 +216,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro if sub.Type == COLLECTIONS { log.Tracef("collection, recurse") - err = h.DownloadLatest(&val, overwrite, updateOnly) + err = h.DownloadLatest(&val, overwrite, updateOnly, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -224,7 +224,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro downloaded := val.Downloaded - err = h.DownloadItem(&val, overwrite) + err = h.DownloadItem(&val, overwrite, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -240,7 +240,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro h.Items[sub.Type][sub.Name] = val } - err = h.DownloadItem(target, overwrite) + err = h.DownloadItem(target, overwrite, hubURLTemplate, branch) if err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -248,7 +248,8 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro return nil } -func (h *Hub) DownloadItem(target *Item, overwrite bool) error { +func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch string) error { + url := fmt.Sprintf(hubURLTemplate, branch, target.RemotePath) tdir := h.cfg.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files @@ -264,7 +265,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { } } - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 1ece43269ac..234e54caa83 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -17,7 +17,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -33,7 +33,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - hub, err := InitHubUpdate(hub.cfg) + hub, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) hub = getHubOrFail(t, hub.cfg) @@ -43,7 +43,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -61,7 +61,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -81,10 +81,10 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - hub, err = InitHubUpdate(hub.cfg) + hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) hub = getHubOrFail(t, hub.cfg) @@ -110,7 +110,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -136,13 +136,13 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - hub, err = InitHubUpdate(hub.cfg) + hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) hub = getHubOrFail(t, hub.cfg) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) hub = getHubOrFail(t, hub.cfg) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 2a2bfbb0a0b..3adf1c308ca 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -15,7 +15,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -const HubIndexFile = ".index.json" +// const HubIndexFile = ".index.json" // Hub represents the runtime status of the hub (parsed items, etc.) type Hub struct { @@ -78,12 +78,12 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { // InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk // It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { +func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) (*Hub, error) { if cfg == nil { return nil, fmt.Errorf("no configuration found for hub") } - bidx, err := DownloadIndex(cfg.HubIndexFile) + bidx, err := DownloadIndex(cfg.HubIndexFile, urlTemplate, branch, remotePath) if err != nil { return nil, fmt.Errorf("failed to download index: %w", err) } @@ -108,10 +108,11 @@ func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { } // DownloadIndex downloads the latest version of the index and returns the content -func DownloadIndex(indexPath string) ([]byte, error) { - log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) +func DownloadIndex(localPath, hubURLTemplate, branch, remotePath string) ([]byte, error) { + url := fmt.Sprintf(hubURLTemplate, branch, remotePath) + log.Debugf("fetching index from branch %s (%s)", branch, url) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("failed to build request for hub index: %w", err) } @@ -135,17 +136,17 @@ func DownloadIndex(indexPath string) ([]byte, error) { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } - oldContent, err := os.ReadFile(indexPath) + oldContent, err := os.ReadFile(localPath) if err != nil { if !os.IsNotExist(err) { log.Warningf("failed to read hub index: %s", err) } } else if bytes.Equal(body, oldContent) { log.Info("hub index is up to date") - // write it anyway, can't hurt + return body, nil } - file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return nil, fmt.Errorf("while opening hub index file: %w", err) @@ -157,7 +158,7 @@ func DownloadIndex(indexPath string) ([]byte, error) { return nil, fmt.Errorf("while writing hub index file: %w", err) } - log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) + log.Infof("Wrote index to %s, %d bytes", localPath, wsize) return body, nil } @@ -209,7 +210,7 @@ func ParseIndex(buff []byte) (HubItems, error) { } // ItemStats returns total counts of the hub items -func (h Hub) ItemStats() []string { +func (h *Hub) ItemStats() []string { loaded := "" for _, itemType := range ItemTypes { // ensure the order is always the same diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 1b3a31109dd..02c552c4c0c 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,7 +13,7 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - _, err := InitHubUpdate(hub.cfg) + _, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err) _, err = GetHub() @@ -21,7 +21,6 @@ func TestInitHubUpdate(t *testing.T) { } func TestDownloadIndex(t *testing.T) { - back := RawFileURLTemplate // bad url template fmt.Println("Test 'bad URL'") @@ -32,9 +31,7 @@ func TestDownloadIndex(t *testing.T) { os.Remove(tmpIndex.Name()) }) - RawFileURLTemplate = "x" - - ret, err := DownloadIndex(tmpIndex.Name()) + ret, err := DownloadIndex(tmpIndex.Name(), "x", "", "") cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ") fmt.Printf("->%+v", ret) @@ -42,9 +39,7 @@ func TestDownloadIndex(t *testing.T) { // bad domain fmt.Println("Test 'bad domain'") - RawFileURLTemplate = "https://baddomain/%s/%s" - - ret, err = DownloadIndex(tmpIndex.Name()) + ret, err = DownloadIndex(tmpIndex.Name(), "https://baddomain/%s/%s", "master", ".index.json") cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") fmt.Printf("->%+v", ret) @@ -52,12 +47,8 @@ func TestDownloadIndex(t *testing.T) { // bad target path fmt.Println("Test 'bad target path'") - RawFileURLTemplate = back - - ret, err = DownloadIndex("/does/not/exist/index.json") + ret, err = DownloadIndex("/does/not/exist/index.json", mockURLTemplate, "master", ".index.json") cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") - RawFileURLTemplate = back - fmt.Printf("->%+v", ret) } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index a9360a93c25..24f948d3c26 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -106,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) // log.Infof("%s -> name:%s stage:%s", path, fname, stage) - + if ret.stage == SCENARIOS { ret.ftype = SCENARIOS ret.stage = "" @@ -394,6 +394,12 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { log.Errorf("failed %s : %s", cpath, err) } + // explicit check for non existing directory, avoid spamming log.Debug + if _, err := os.Stat(cpath); os.IsNotExist(err) { + log.Tracef("directory %s doesn't exist, skipping", cpath) + continue + } + err = filepath.WalkDir(cpath, h.itemVisit) if err != nil { return warnings, err diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 347340253d4..0c38db361c3 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -46,14 +46,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { +func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubURLTemplate, branch string) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err } - cwhub.SetHubBranch() - hub, err := cwhub.InitHub(csConfig.Hub) if err != nil { return fmt.Errorf("getting hub index: %w", err) @@ -76,7 +74,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing collection %s: %w", collection, err) } } @@ -90,7 +88,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing parser %s: %w", parser, err) } } @@ -104,7 +102,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing scenario %s: %w", scenario, err) } } @@ -118,7 +116,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } } From 495c6f9e8a4574b27c546bdb9b373b2b821f206f Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 11:08:54 +0200 Subject: [PATCH 135/263] add debug to rule collection --- pkg/waf/waap_rules_collection.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 02cd12e87f3..3e6e2fbbe1b 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -26,6 +26,7 @@ var WAAP_RULE = "waap-rule" type WaapCollectionConfig struct { Type string `yaml:"type"` Name string `yaml:"name"` + Debug bool `yaml:"debug"` Description string `yaml:"description"` SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` @@ -70,6 +71,11 @@ func LoadCollection(collection string) (WaapCollection, error) { continue } log.Infof("Adding %s to waap rules", rule.Name) + // if rule.Debug { + // log.Infof("Enabling debug for collection %s", rule.Name) + + // //SetRuleDebug(rule.ID, true) + // } waapRules[rule.Name] = rule } From 31a3b8a4efe7812379ff11d04c82d6ca9528ddcc Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 11:09:19 +0200 Subject: [PATCH 136/263] move this to pkg/waf --- .../modules/waap => waf}/coraza_logger.go | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) rename pkg/{acquisition/modules/waap => waf}/coraza_logger.go (88%) diff --git a/pkg/acquisition/modules/waap/coraza_logger.go b/pkg/waf/coraza_logger.go similarity index 88% rename from pkg/acquisition/modules/waap/coraza_logger.go rename to pkg/waf/coraza_logger.go index f9a5ee42784..a7302e033d5 100644 --- a/pkg/acquisition/modules/waap/coraza_logger.go +++ b/pkg/waf/coraza_logger.go @@ -1,4 +1,4 @@ -package wafacquisition +package waf import ( "fmt" @@ -8,6 +8,19 @@ import ( log "github.com/sirupsen/logrus" ) +var DebugRules map[int]bool = map[int]bool{} + +func SetRuleDebug(id int, debug bool) { + DebugRules[id] = debug +} + +func GetRuleDebug(id int) bool { + if val, ok := DebugRules[id]; ok { + return val + } + return false +} + // type ContextField func(Event) Event type crzLogEvent struct { @@ -56,7 +69,15 @@ func (e *crzLogEvent) Bool(key string, b bool) dbg.Event { func (e *crzLogEvent) Int(key string, i int) dbg.Event { if e.muted { - return e + if key == "rule_id" { + log.Warningf("is rule_id %d in debug mode -> %t", i, GetRuleDebug(i)) + if GetRuleDebug(i) { + e.muted = false + e.fields = map[string]interface{}{} + } else { + return e + } + } } //e.logger.Info("int") e.fields[key] = i @@ -136,7 +157,7 @@ func (c crzLogger) Trace() dbg.Event { func (c crzLogger) Debug() dbg.Event { if c.logLevel < log.DebugLevel { //c.logger.Infof("ignoring debug directive -> %s", c.logLevel.String()) - return &crzLogEvent{muted: true} + return &crzLogEvent{muted: true, logger: c.logger} } From bb59d9852a7e7cdb5030c8d6bfeb045fadd82622 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 11:09:38 +0200 Subject: [PATCH 137/263] make Event viabl --- pkg/acquisition/modules/waap/utils.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index f33ecc341e5..253193018ab 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -88,11 +88,11 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "coraza-waf"}, + Labels: map[string]string{"type": "coraza-waap"}, Process: true, - Module: "waf", - Src: "waf", - Raw: "dummy-waf-data", //we discard empty Line.Raw items :) + Module: "waap", + Src: "waap", + Raw: "dummy-waap-data", //we discard empty Line.Raw items :) } evt.Waap = types.WaapEvent{} @@ -144,6 +144,9 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest if evt.Meta == nil { evt.Meta = map[string]string{} } + if evt.Parsed == nil { + evt.Parsed = map[string]string{} + } if req.IsInBand { evt.Meta["waap_interrupted"] = "true" evt.Meta["waap_action"] = req.Tx.Interruption().Action From 01ddc45a2c0f6c18612dfc62af191f2986307516 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 11:09:56 +0200 Subject: [PATCH 138/263] use loggeR --- pkg/acquisition/modules/waap/waap_runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index a986f5f811e..d91af6ac320 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -43,7 +43,7 @@ func (r *WaapRunner) Init(datadir string) error { runnerLogger := r.logger.Dup() //setting up inband engine - inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)) + inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(runnerLogger)) if !r.WaapRuntime.Config.InbandOptions.DisableBodyInspection { inbandCfg = inbandCfg.WithRequestBodyAccess() } else { @@ -58,7 +58,7 @@ func (r *WaapRunner) Init(datadir string) error { } //setting up outband engine - outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(NewCrzLogger(runnerLogger)) + outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(runnerLogger)) if !r.WaapRuntime.Config.OutOfBandOptions.DisableBodyInspection { outbandCfg = outbandCfg.WithRequestBodyAccess() } else { From e5906e6eea1fa91b5a1dd180e25a48a560900977 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 27 Oct 2023 11:10:17 +0200 Subject: [PATCH 139/263] up --- pkg/acquisition/modules/waap/utils.go | 22 ++++++++++++-- pkg/acquisition/modules/waap/waap_runner.go | 7 +++++ pkg/waf/waap_rules_collection.go | 33 +++++++++++++++++++-- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 253193018ab..4223bc161da 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -11,6 +11,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -196,6 +197,21 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest } WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() + spew.Dump(waf.WaapRulesDetails) + + name := "NOT_SET" + version := "NOT_SET" + hash := "NOT_SET" + + if details, ok := waf.WaapRulesDetails[rule.Rule().ID()]; ok { + //Only set them for custom rules, not for rules written in seclang + name = details.Name + version = details.Version + hash = details.Hash + + r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) + } + corazaRule := map[string]interface{}{ "id": rule.Rule().ID(), "uri": evt.Parsed["uri"], @@ -210,9 +226,9 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest "accuracy": rule.Rule().Accuracy(), "msg": rule.Message(), "severity": rule.Rule().Severity().String(), - "name": "FIXFIXFIXFIXFIX", - "hash": "FIXIFIX", - "version": "FIXFIXFIX", + "name": name, + "hash": hash, + "version": version, } evt.Waap.MatchedRules = append(evt.Waap.MatchedRules, corazaRule) } diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index d91af6ac320..7867ad04763 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -92,6 +92,13 @@ func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *wa }() request.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) + + for k, v := range request.Args { + for _, vv := range v { + request.Tx.AddGetRequestArgument(k, vv) + } + } + request.Tx.ProcessURI(request.URI, request.Method, request.Proto) //TODO: The doc mentions that GET args needs to be added, but we never call AddArguments ? for k, vr := range request.Headers { diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 3e6e2fbbe1b..39dc7b614a5 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -31,9 +31,23 @@ type WaapCollectionConfig struct { SecLangFilesRules []string `yaml:"seclang_files_rules"` SecLangRules []string `yaml:"seclang_rules"` Rules []waap_rule.CustomRule `yaml:"rules"` - Data interface{} `yaml:"data"` //Ignore it + + Data interface{} `yaml:"data"` //Ignore it + hash string `yaml:"-"` + version string `yaml:"-"` +} + +type RulesDetails struct { + LogLevel log.Level + Hash string + Version string + Name string } +// Should it be a global ? +// Is using the id is a good idea ? might be too specific to coraza and not easily reusable +var WaapRulesDetails = make(map[int]RulesDetails) + func LoadCollection(collection string) (WaapCollection, error) { //FIXME: do it once globally @@ -70,6 +84,10 @@ func LoadCollection(collection string) (WaapCollection, error) { log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubWafRuleItem.LocalPath) continue } + + rule.hash = hubWafRuleItem.LocalHash + rule.version = hubWafRuleItem.Version + log.Infof("Adding %s to waap rules", rule.Name) // if rule.Debug { // log.Infof("Enabling debug for collection %s", rule.Name) @@ -122,13 +140,24 @@ func LoadCollection(collection string) (WaapCollection, error) { if loadedRule.Rules != nil { for _, rule := range loadedRule.Rules { - strRule, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) + strRule, ruleId, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) if err != nil { log.Errorf("unable to convert rule %s : %s", rule.Name, err) return WaapCollection{}, err } log.Infof("Adding rule %s", strRule) waapCol.Rules = append(waapCol.Rules, strRule) + + if _, ok := WaapRulesDetails[int(ruleId)]; !ok { + WaapRulesDetails[int(ruleId)] = RulesDetails{ + LogLevel: log.InfoLevel, + Hash: loadedRule.hash, + Version: loadedRule.version, + Name: loadedRule.Name, + } + } else { + log.Warnf("conflicting id %d for rule %s !", ruleId, rule.Name) + } } } From b0e7da06b99d2268e39e7b7bd9ad57a8b6385af8 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 27 Oct 2023 11:10:35 +0200 Subject: [PATCH 140/263] up --- pkg/waf/request.go | 9 ++++- pkg/waf/waap_rule/modsec_rule_test.go | 52 ++++++++++++++++++++++++++- pkg/waf/waap_rule/modsecurity.go | 31 ++++++++++------ pkg/waf/waap_rule/waap_rule.go | 13 +++---- 4 files changed, 87 insertions(+), 18 deletions(-) diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 49fcfe45134..5a7cc04a3a2 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -64,6 +64,7 @@ type ParsedRequest struct { Host string ClientIP string URI string + Args url.Values ClientHost string Headers http.Header URL *url.URL @@ -117,18 +118,24 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { delete(r.Header, URIHeaderName) delete(r.Header, VerbHeaderName) + parsedURL, err := url.Parse(clientURI) + if err != nil { + return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err) + } + return ParsedRequest{ RemoteAddr: r.RemoteAddr, UUID: uuid.New().String(), ClientHost: clientHost, ClientIP: clientIP, - URI: clientURI, + URI: parsedURL.Path, Method: clientMethod, Host: r.Host, Headers: r.Header, URL: r.URL, Proto: r.Proto, Body: body, + Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args TransferEncoding: r.TransferEncoding, ResponseChannel: make(chan WaapTempResponse), }, nil diff --git a/pkg/waf/waap_rule/modsec_rule_test.go b/pkg/waf/waap_rule/modsec_rule_test.go index 3f40eff6949..62e04d8e154 100644 --- a/pkg/waf/waap_rule/modsec_rule_test.go +++ b/pkg/waf/waap_rule/modsec_rule_test.go @@ -50,11 +50,61 @@ func TestVPatchRuleString(t *testing.T) { expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2323451654,phase:2,deny,log,msg:'Basic AND_and_0',t:lowercase,chain" SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2075918819,phase:2,deny,log,msg:'Basic AND_and_1',t:lowercase"`, }, + { + name: "Basic OR", + rule: CustomRule{ + Or: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2720972114,phase:2,deny,log,msg:'Basic OR_or_0',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2638639999,phase:2,deny,log,msg:'Basic OR_or_1',t:lowercase"`, + }, + { + name: "OR AND mix", + rule: CustomRule{ + And: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + Or: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2720972114,phase:2,deny,log,msg:'Basic OR_or_0',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2638639999,phase:2,deny,log,msg:'Basic OR_or_1',t:lowercase"`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual, err := tt.rule.Convert(ModsecurityRuleType, tt.name) + actual, _, err := tt.rule.Convert(ModsecurityRuleType, tt.name) if err != nil { t.Errorf("Error converting rule: %s", err) diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go index 438cea2eff9..3753234b78a 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/waf/waap_rule/modsecurity.go @@ -7,6 +7,7 @@ import ( ) type ModsecurityRule struct { + id uint32 } var zonesMap map[string]string = map[string]string{ @@ -42,18 +43,19 @@ var matchMap map[string]string = map[string]string{ "le": "@le", } -func (m ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, error) { +func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, uint32, error) { - rules, err := m.buildRules(rule, waapRuleName, false) + rules, err := m.buildRules(rule, waapRuleName, false, 0) if err != nil { - return "", err + return "", 0, err } - return strings.Join(rules, "\n"), nil + //We return the id of the first generated rule, as it's the interesting one in case of chain or skip + return strings.Join(rules, "\n"), m.id, nil } -func (m ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) uint32 { +func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) uint32 { h := fnv.New32a() h.Write([]byte(waapRuleName)) h.Write([]byte(rule.Match.Type)) @@ -64,17 +66,21 @@ func (m ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) u for _, transform := range rule.Transform { h.Write([]byte(transform)) } - return h.Sum32() + id := h.Sum32() + if m.id == 0 { + m.id = id + } + return id } -func (m ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool) ([]string, error) { +func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool, toSkip int) ([]string, error) { ret := make([]string, 0) if rule.And != nil { for c, andRule := range rule.And { subName := fmt.Sprintf("%s_and_%d", waapRuleName, c) - lastRule := c == len(rule.And)-1 - rules, err := m.buildRules(&andRule, subName, !lastRule) + lastRule := c == len(rule.And)-1 // || len(rule.Or) == 0 + rules, err := m.buildRules(&andRule, subName, !lastRule, 0) if err != nil { return nil, err } @@ -85,7 +91,8 @@ func (m ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and b if rule.Or != nil { for c, orRule := range rule.Or { subName := fmt.Sprintf("%s_or_%d", waapRuleName, c) - rules, err := m.buildRules(&orRule, subName, false) + skip := len(rule.Or) - c - 1 + rules, err := m.buildRules(&orRule, subName, false, skip) if err != nil { return nil, err } @@ -145,6 +152,10 @@ func (m ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and b r.WriteString(",chain") } + if toSkip > 0 { + r.WriteString(fmt.Sprintf(",skip:%d", toSkip)) + } + r.WriteByte('"') ret = append(ret, r.String()) diff --git a/pkg/waf/waap_rule/waap_rule.go b/pkg/waf/waap_rule/waap_rule.go index 00d3210d18e..3dbad290365 100644 --- a/pkg/waf/waap_rule/waap_rule.go +++ b/pkg/waf/waap_rule/waap_rule.go @@ -42,24 +42,25 @@ type CustomRule struct { Or []CustomRule `yaml:"or,omitempty"` } -func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, error) { +func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, uint32, error) { if v.Zones == nil && v.And == nil && v.Or == nil { - return "", fmt.Errorf("no zones defined") + return "", 0, fmt.Errorf("no zones defined") } if v.Match.Type == "" && v.And == nil && v.Or == nil { - return "", fmt.Errorf("no match type defined") + return "", 0, fmt.Errorf("no match type defined") } if v.Match.Value == "" && v.And == nil && v.Or == nil { - return "", fmt.Errorf("no match value defined") + return "", 0, fmt.Errorf("no match value defined") } switch ruleType { case ModsecurityRuleType: - return ModsecurityRule{}.Build(v, waapRuleName) + r := ModsecurityRule{} + return r.Build(v, waapRuleName) default: - return "", fmt.Errorf("unknown rule format '%s'", ruleType) + return "", 0, fmt.Errorf("unknown rule format '%s'", ruleType) } } From 37c5d54e431f2436b0d952058f6ed48ba02d9a32 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 27 Oct 2023 11:17:27 +0200 Subject: [PATCH 141/263] up --- pkg/waf/waap_rule/modsecurity.go | 12 +++++------- pkg/waf/waap_rule/waap_rule.go | 10 +++++----- pkg/waf/waap_rules_collection.go | 13 +++++++++---- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go index 3753234b78a..b7ff3242807 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/waf/waap_rule/modsecurity.go @@ -7,7 +7,7 @@ import ( ) type ModsecurityRule struct { - id uint32 + ids []uint32 } var zonesMap map[string]string = map[string]string{ @@ -43,16 +43,16 @@ var matchMap map[string]string = map[string]string{ "le": "@le", } -func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, uint32, error) { +func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, []uint32, error) { rules, err := m.buildRules(rule, waapRuleName, false, 0) if err != nil { - return "", 0, err + return "", nil, err } //We return the id of the first generated rule, as it's the interesting one in case of chain or skip - return strings.Join(rules, "\n"), m.id, nil + return strings.Join(rules, "\n"), m.ids, nil } func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) uint32 { @@ -67,9 +67,7 @@ func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) h.Write([]byte(transform)) } id := h.Sum32() - if m.id == 0 { - m.id = id - } + m.ids = append(m.ids, id) return id } diff --git a/pkg/waf/waap_rule/waap_rule.go b/pkg/waf/waap_rule/waap_rule.go index 3dbad290365..279ff0a0bc9 100644 --- a/pkg/waf/waap_rule/waap_rule.go +++ b/pkg/waf/waap_rule/waap_rule.go @@ -42,18 +42,18 @@ type CustomRule struct { Or []CustomRule `yaml:"or,omitempty"` } -func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, uint32, error) { +func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, []uint32, error) { if v.Zones == nil && v.And == nil && v.Or == nil { - return "", 0, fmt.Errorf("no zones defined") + return "", nil, fmt.Errorf("no zones defined") } if v.Match.Type == "" && v.And == nil && v.Or == nil { - return "", 0, fmt.Errorf("no match type defined") + return "", nil, fmt.Errorf("no match type defined") } if v.Match.Value == "" && v.And == nil && v.Or == nil { - return "", 0, fmt.Errorf("no match value defined") + return "", nil, fmt.Errorf("no match value defined") } switch ruleType { @@ -61,6 +61,6 @@ func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, uint r := ModsecurityRule{} return r.Build(v, waapRuleName) default: - return "", 0, fmt.Errorf("unknown rule format '%s'", ruleType) + return "", nil, fmt.Errorf("unknown rule format '%s'", ruleType) } } diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 39dc7b614a5..d78926897b6 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -140,7 +140,7 @@ func LoadCollection(collection string) (WaapCollection, error) { if loadedRule.Rules != nil { for _, rule := range loadedRule.Rules { - strRule, ruleId, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) + strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) if err != nil { log.Errorf("unable to convert rule %s : %s", rule.Name, err) return WaapCollection{}, err @@ -148,15 +148,20 @@ func LoadCollection(collection string) (WaapCollection, error) { log.Infof("Adding rule %s", strRule) waapCol.Rules = append(waapCol.Rules, strRule) - if _, ok := WaapRulesDetails[int(ruleId)]; !ok { - WaapRulesDetails[int(ruleId)] = RulesDetails{ + //We only take the first id, as it's the one of the "main" rule + if _, ok := WaapRulesDetails[int(rulesId[0])]; !ok { + WaapRulesDetails[int(rulesId[0])] = RulesDetails{ LogLevel: log.InfoLevel, Hash: loadedRule.hash, Version: loadedRule.version, Name: loadedRule.Name, } } else { - log.Warnf("conflicting id %d for rule %s !", ruleId, rule.Name) + log.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) + } + + for _, id := range rulesId { + SetRuleDebug(int(id), loadedRule.Debug) } } } From 57b5f5c27c2172140bf24894f5948b6ac26978ba Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 27 Oct 2023 11:21:19 +0200 Subject: [PATCH 142/263] uip --- pkg/waf/coraza_logger.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/waf/coraza_logger.go b/pkg/waf/coraza_logger.go index a7302e033d5..5630651df78 100644 --- a/pkg/waf/coraza_logger.go +++ b/pkg/waf/coraza_logger.go @@ -77,6 +77,8 @@ func (e *crzLogEvent) Int(key string, i int) dbg.Event { } else { return e } + } else { + return e } } //e.logger.Info("int") From c96c8f19c94aaf91a3a96ab3fec3a346fd0b27a6 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 16:07:25 +0200 Subject: [PATCH 143/263] logging clean up --- pkg/acquisition/modules/waap/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index 4223bc161da..da25daa6f89 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -46,7 +46,6 @@ func WaapEventGeneration(inEvt types.Event) (types.Event, error) { Value: string(valueByte), } alert.Meta = append(alert.Meta, &meta) - log.Infof("adding Meta - %s = %s", key, inEvt.Parsed[key]) } alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"waf"} //don't know what to do about this From 83d52111939acc9315a9e844df4ea8ea6905a87a Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 16:07:37 +0200 Subject: [PATCH 144/263] logging clean up --- pkg/acquisition/modules/waap/waap.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index a8b6fd411a6..3089a2ce1e8 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -269,9 +269,6 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { waapResponse := w.WaapRuntime.GenerateResponse(response.InBandInterrupt) - log.Infof("resp %+v", response) - log.Infof("waap resp %+v", waapResponse) - rw.WriteHeader(waapResponse.HTTPStatus) body, err := json.Marshal(BodyResponse{Action: waapResponse.Action}) if err != nil { From 81645c96aab9f8f983ddcd50d4e00f6ec2ec0381 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 16:07:49 +0200 Subject: [PATCH 145/263] logging clean up --- pkg/acquisition/modules/waap/waap_runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 7867ad04763..5bf8cef636d 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -142,7 +142,7 @@ func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *wa } if in != nil { - r.logger.Infof("rules matched for body : %d", in.RuleID) + r.logger.Debugf("rules matched for body : %d", in.RuleID) return nil } @@ -169,7 +169,7 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Infof("Waf Runner is dying") return nil case request := <-r.inChan: - r.logger.Infof("Requests handled by runner %s", request.UUID) + r.logger.Debugf("Requests handled by runner %s", request.UUID) r.WaapRuntime.ClearResponse() request.IsInBand = true From d136cc47348eed1fe9e701b140352c3ca50086fa Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 16:10:36 +0200 Subject: [PATCH 146/263] logging clean up --- pkg/waf/coraza_logger.go | 109 +++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/pkg/waf/coraza_logger.go b/pkg/waf/coraza_logger.go index 5630651df78..82e0a169774 100644 --- a/pkg/waf/coraza_logger.go +++ b/pkg/waf/coraza_logger.go @@ -27,16 +27,32 @@ type crzLogEvent struct { fields log.Fields logger *log.Entry muted bool + level log.Level } func (e *crzLogEvent) Msg(msg string) { if e.muted { return } + + /*this is a hack. As we want to have per-level rule debug but it's not allowed by coraza/modsec, if a rule ID is flagged to be in debug mode, the + .Int("rule_id", ) call will set the log_level of the event to debug. However, given the logger is global to the waap-runner, + we are switching forth and back the log level of the logger*/ + oldLvl := e.logger.Logger.GetLevel() + + if e.level != oldLvl { + e.logger.Logger.SetLevel(e.level) + } + if len(e.fields) == 0 { - e.logger.Info(msg) + e.logger.Log(e.level, msg) } else { - e.logger.WithFields(e.fields).Info(msg) + e.logger.WithFields(e.fields).Log(e.level, msg) + } + + if e.level != oldLvl { + e.logger.Logger.SetLevel(oldLvl) + e.level = oldLvl } } @@ -44,7 +60,6 @@ func (e *crzLogEvent) Str(key, val string) dbg.Event { if e.muted { return e } - //e.logger.Info("str") e.fields[key] = val return e } @@ -53,7 +68,6 @@ func (e *crzLogEvent) Err(err error) dbg.Event { if e.muted { return e } - //e.logger.Info("err") e.fields["error"] = err return e } @@ -62,26 +76,21 @@ func (e *crzLogEvent) Bool(key string, b bool) dbg.Event { if e.muted { return e } - //e.logger.Info("bool") e.fields[key] = b return e } func (e *crzLogEvent) Int(key string, i int) dbg.Event { if e.muted { - if key == "rule_id" { - log.Warningf("is rule_id %d in debug mode -> %t", i, GetRuleDebug(i)) - if GetRuleDebug(i) { - e.muted = false - e.fields = map[string]interface{}{} - } else { - return e - } + //this allows us to have per-rule debug logging + if key == "rule_id" && GetRuleDebug(i) { + e.muted = false + e.fields = map[string]interface{}{} + e.level = log.DebugLevel } else { return e } } - //e.logger.Info("int") e.fields[key] = i return e } @@ -90,7 +99,6 @@ func (e *crzLogEvent) Uint(key string, i uint) dbg.Event { if e.muted { return e } - //e.logger.Info("uint") e.fields[key] = i return e } @@ -99,7 +107,6 @@ func (e *crzLogEvent) Stringer(key string, val fmt.Stringer) dbg.Event { if e.muted { return e } - //e.logger.Info("stringer") e.fields[key] = val return e } @@ -121,20 +128,31 @@ func NewCrzLogger(logger *log.Entry) crzLogger { return crzLogger{logger: logger, logLevel: logger.Logger.GetLevel()} } +func (c crzLogger) NewMutedEvt(lvl log.Level) dbg.Event { + return &crzLogEvent{muted: true, logger: c.logger, level: lvl} +} +func (c crzLogger) NewEvt(lvl log.Level) dbg.Event { + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger, level: lvl} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + func (c crzLogger) WithOutput(w io.Writer) dbg.Logger { - c.logger.Infof("ignoring withoutput directive") return c } func (c crzLogger) WithLevel(lvl dbg.Level) dbg.Logger { - c.logger.Warningf("setting log level to %s", lvl) c.logLevel = log.Level(lvl) c.logger.Logger.SetLevel(c.logLevel) return c } func (c crzLogger) With(fs ...dbg.ContextField) dbg.Logger { - var e dbg.Event = &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} + var e dbg.Event = c.NewEvt(c.logLevel) for _, f := range fs { e = f(e) } @@ -144,69 +162,36 @@ func (c crzLogger) With(fs ...dbg.ContextField) dbg.Logger { func (c crzLogger) Trace() dbg.Event { if c.logLevel < log.TraceLevel { - //c.logger.Infof("ignoring trace directive") - return &crzLogEvent{muted: true} + return c.NewMutedEvt(log.TraceLevel) } - evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} - if c.defaultFields != nil { - for k, v := range c.defaultFields { - evt.fields[k] = v - } - } - return evt + return c.NewEvt(log.TraceLevel) } func (c crzLogger) Debug() dbg.Event { if c.logLevel < log.DebugLevel { - //c.logger.Infof("ignoring debug directive -> %s", c.logLevel.String()) - return &crzLogEvent{muted: true, logger: c.logger} - - } + return c.NewMutedEvt(log.DebugLevel) - evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} - if c.defaultFields != nil { - for k, v := range c.defaultFields { - evt.fields[k] = v - } } - return evt + return c.NewEvt(log.DebugLevel) } func (c crzLogger) Info() dbg.Event { if c.logLevel < log.InfoLevel { - return &crzLogEvent{muted: true} - } - evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} - if c.defaultFields != nil { - for k, v := range c.defaultFields { - evt.fields[k] = v - } + return c.NewMutedEvt(log.InfoLevel) } - return evt + return c.NewEvt(log.InfoLevel) } func (c crzLogger) Warn() dbg.Event { if c.logLevel < log.WarnLevel { - return &crzLogEvent{muted: true} + return c.NewMutedEvt(log.WarnLevel) } - evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} - if c.defaultFields != nil { - for k, v := range c.defaultFields { - evt.fields[k] = v - } - } - return evt + return c.NewEvt(log.WarnLevel) } func (c crzLogger) Error() dbg.Event { if c.logLevel < log.ErrorLevel { - return &crzLogEvent{muted: true} + return c.NewMutedEvt(log.ErrorLevel) } - evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger} - if c.defaultFields != nil { - for k, v := range c.defaultFields { - evt.fields[k] = v - } - } - return evt + return c.NewEvt(log.ErrorLevel) } From 2e0b9683f31c2f54e9a07d94542434c4a6698eb0 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 27 Oct 2023 16:10:46 +0200 Subject: [PATCH 147/263] logging clean up --- pkg/waf/waap_rules_collection.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index d78926897b6..45e5995b3f2 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -89,11 +89,7 @@ func LoadCollection(collection string) (WaapCollection, error) { rule.version = hubWafRuleItem.Version log.Infof("Adding %s to waap rules", rule.Name) - // if rule.Debug { - // log.Infof("Enabling debug for collection %s", rule.Name) - // //SetRuleDebug(rule.ID, true) - // } waapRules[rule.Name] = rule } @@ -108,8 +104,6 @@ func LoadCollection(collection string) (WaapCollection, error) { return WaapCollection{}, fmt.Errorf("no waap rules found for collection %s", collection) } - log.Infof("Found rule collection %s with %+v", loadedRule.Name, loadedRule) - waapCol := WaapCollection{ collectionName: loadedRule.Name, } From 17662e59a9c7443979a6d1878777d5867931e3be Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:23:50 +0100 Subject: [PATCH 148/263] Refact pkg/cwhub, cscli: hub upgrades (#2568) * fix bats test for "upgrade all items" * refact UpgradeConfig() -> UpgradeItem(): one item only * refact RemoveMany() -> RemoveItem() * Computed value: Item.Local -> Item.IsLocal() * refact url/branch configuration with LocalHubCfg/RemoteHubCfg --- cmd/crowdsec-cli/branch.go | 46 ------- cmd/crowdsec-cli/capi.go | 2 +- cmd/crowdsec-cli/config_backup.go | 29 ++-- cmd/crowdsec-cli/config_restore.go | 24 ++-- cmd/crowdsec-cli/console.go | 2 +- cmd/crowdsec-cli/hub.go | 77 ++++------- cmd/crowdsec-cli/item_metrics.go | 1 + cmd/crowdsec-cli/item_suggest.go | 9 +- cmd/crowdsec-cli/itemcommands.go | 106 +++++++++++---- cmd/crowdsec-cli/items.go | 51 +++---- cmd/crowdsec-cli/lapi.go | 6 +- cmd/crowdsec-cli/main.go | 9 +- cmd/crowdsec-cli/require/branch.go | 61 +++++++++ cmd/crowdsec-cli/require/require.go | 23 +++- cmd/crowdsec-cli/setup.go | 11 +- cmd/crowdsec-cli/simulation.go | 2 +- cmd/crowdsec-cli/support.go | 14 +- cmd/crowdsec-cli/utils_table.go | 29 ++-- cmd/crowdsec/crowdsec.go | 4 +- pkg/csconfig/config.go | 2 +- pkg/csconfig/config_test.go | 4 +- pkg/csconfig/hub.go | 6 +- pkg/csconfig/hub_test.go | 4 +- pkg/cwhub/cwhub_test.go | 20 ++- pkg/cwhub/dataset_test.go | 11 +- pkg/cwhub/enable.go | 20 +-- pkg/cwhub/enable_test.go | 4 +- pkg/cwhub/helpers.go | 189 +++++++++++--------------- pkg/cwhub/helpers_test.go | 67 ++++++--- pkg/cwhub/hub.go | 57 +++++--- pkg/cwhub/hub_test.go | 39 +++++- pkg/cwhub/items.go | 58 ++++++-- pkg/cwhub/items_test.go | 8 +- pkg/cwhub/leakybucket.go | 4 +- pkg/cwhub/sync.go | 22 +-- pkg/hubtest/hubtest_item.go | 28 ++-- pkg/leakybucket/buckets_test.go | 4 +- pkg/parser/unix_parser.go | 5 +- pkg/setup/install.go | 16 +-- test/bats/01_crowdsec.bats | 4 +- test/bats/20_hub.bats | 78 +++++++++++ test/bats/20_hub_collections.bats | 64 +++++---- test/bats/20_hub_collections_dep.bats | 6 +- test/bats/20_hub_parsers.bats | 153 +++++---------------- test/bats/20_hub_postoverflows.bats | 64 +++++---- test/bats/20_hub_scenarios.bats | 60 ++++---- test/lib/setup_file.sh | 22 ++- 47 files changed, 847 insertions(+), 678 deletions(-) delete mode 100644 cmd/crowdsec-cli/branch.go create mode 100644 cmd/crowdsec-cli/require/branch.go create mode 100644 test/bats/20_hub.bats diff --git a/cmd/crowdsec-cli/branch.go b/cmd/crowdsec-cli/branch.go deleted file mode 100644 index f4dae2947fe..00000000000 --- a/cmd/crowdsec-cli/branch.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -// Set the appropriate hub branch according to config settings and crowdsec version - -import ( - log "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" - - "github.com/crowdsecurity/crowdsec/pkg/cwversion" -) - -// chooseHubBranch returns the branch name to use for the hub -// It can be "master" or the branch corresponding to the current crowdsec version -func chooseHubBranch() string { - latest, err := cwversion.Latest() - if err != nil { - log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - return "master" - } - - csVersion := cwversion.VersionStrip() - if csVersion == latest { - log.Debugf("current version is equal to latest (%s)", csVersion) - return "master" - } - - // if current version is greater than the latest we are in pre-release - if semver.Compare(csVersion, latest) == 1 { - log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion) - return "master" - } - - if csVersion == "" { - log.Warning("Crowdsec version is not set, using master branch for the hub") - return "master" - } - - log.Warnf("Crowdsec is not the latest version. "+ - "Current version is '%s' and the latest stable version is '%s'. Please update it!", - csVersion, latest) - - log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+ - "added to Crowdsec Hub after CrowdSec %s", latest) - - return csVersion -} diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 3f94d286c84..69fc0630dc9 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -151,7 +151,7 @@ func NewCapiStatusCmd() *cobra.Command { return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) } - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 417b21eda31..702ad5b1b1d 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -13,11 +13,15 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) -func backupHub(hub *cwhub.Hub, dirPath string) error { - var err error +func backupHub(dirPath string) error { var itemDirectory string var upstreamParsers []string + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + for _, itemType := range cwhub.ItemTypes { clog := log.WithFields(log.Fields{ "type": itemType, @@ -28,7 +32,7 @@ func backupHub(hub *cwhub.Hub, dirPath string) error { continue } itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType) - if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil { + if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil { return fmt.Errorf("error while creating %s : %s", itemDirectory, err) } upstreamParsers = []string{} @@ -42,15 +46,15 @@ func backupHub(hub *cwhub.Hub, dirPath string) error { } //for the local/tainted ones, we backup the full file - if v.Tainted || v.Local || !v.UpToDate { + if v.Tainted || v.IsLocal() || !v.UpToDate { //we need to backup stages for parsers if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) - if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { + if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) } } - clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) + clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.IsLocal(), v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) if err = CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) @@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) - if _, err := os.Stat(parentDir); err != nil { + if _, err = os.Stat(parentDir); err != nil { return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) } @@ -189,12 +193,7 @@ func backupConfigToDirectory(dirPath string) error { log.Infof("Saved profiles to %s", backupProfiles) } - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - if err = backupHub(hub, dirPath); err != nil { + if err = backupHub(dirPath); err != nil { return fmt.Errorf("failed to backup hub config: %s", err) } @@ -202,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error { } func runConfigBackup(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - if err := backupConfigToDirectory(args[0]); err != nil { return fmt.Errorf("failed to backup config: %w", err) } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index d2b31c1d1c7..3c036684082 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -22,21 +22,16 @@ type OldAPICfg struct { } // it's a rip of the cli version, but in silent-mode -func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, error) { - hub, err := cwhub.GetHub() - if err != nil { - return "", err - } - +func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) { var item = hub.GetItem(obtype, name) if item == nil { return "", fmt.Errorf("error retrieving item") } - err = hub.DownloadLatest(item, false, false, hubURLTemplate, branch) + err := hub.DownloadLatest(item, false, false) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } - if err := hub.AddItem(*item); err != nil { + if err = hub.AddItem(*item); err != nil { return "", err } @@ -51,9 +46,10 @@ func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, err } func restoreHub(dirPath string) error { - var err error - - branch := chooseHubBranch() + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) @@ -73,7 +69,7 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - label, err := silentInstallItem(toinstall, itype, hubURLTemplate, branch) + label, err := silentInstallItem(hub, toinstall, itype) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) } else if label != "" { @@ -297,10 +293,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return err } - if _, err := require.Hub(csConfig); err != nil { - return err - } - if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil { return fmt.Errorf("failed to restore config from %s: %w", args[0], err) } diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 06eae5e56ee..79ba6a053d9 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -71,7 +71,7 @@ After running this command your will need to validate the enrollment in the weba return fmt.Errorf("could not parse CAPI URL: %s", err) } - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { return err } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 6023f0e86ef..79cf4190e2f 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -12,13 +12,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -const ( - hubURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" - remoteIndexPath = ".index.json" -) - -var hubBranch = "" - func NewHubCmd() *cobra.Command { var cmdHub = &cobra.Command{ Use: "hub [action]", @@ -56,7 +49,7 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { return err } @@ -71,7 +64,7 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(line) } - err = ListItems(color.Output, []string{ + err = ListItems(hub, color.Output, []string{ cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, }, nil, true, false, all) if err != nil { @@ -97,23 +90,20 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - // don't use require.Hub because if there is no index file, it would fail - - branch := hubBranch - if branch == "" { - branch = chooseHubBranch() - } + local := csConfig.Hub + remote := require.RemoteHub(csConfig) - log.Debugf("Using branch '%s' for the hub", branch) - - hub, err := cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath) + // don't use require.Hub because if there is no index file, it would fail + hub, err := cwhub.InitHubUpdate(local, remote) if err != nil { + // XXX: this should be done when downloading items, too + // but what is the fallback to master actually solving? if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) } - log.Warnf("Could not find index file for branch '%s', using 'master'", branch) - branch = "master" - if hub, err = cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath); err != nil { + log.Warnf("Could not find index file for branch '%s', using 'master'", remote.Branch) + remote.Branch = "master" + if hub, err = cwhub.InitHubUpdate(local, remote); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } @@ -132,7 +122,7 @@ func NewHubUpdateCmd() *cobra.Command { Use: "update", Short: "Download the latest index (catalog of available configurations)", Long: ` -Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. +Fetches the .index.json file from the hub, containing the list of available configs. `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, @@ -157,36 +147,29 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { return err } - branch := hubBranch - if branch == "" { - branch = chooseHubBranch() - } - - log.Debugf("Using branch '%s' for the hub", branch) - - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) if err != nil { return err } - log.Infof("Upgrading collections") - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force, hubURLTemplate, branch); err != nil { - return err - } - - log.Infof("Upgrading parsers") - if err := hub.UpgradeConfig(cwhub.PARSERS, "", force, hubURLTemplate, branch); err != nil { - return err - } - - log.Infof("Upgrading scenarios") - if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force, hubURLTemplate, branch); err != nil { - return err - } + for _, itemType := range cwhub.ItemTypes { + items, err := hub.GetInstalledItems(itemType) + if err != nil { + return err + } - log.Infof("Upgrading postoverflows") - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force, hubURLTemplate, branch); err != nil { - return err + updated := 0 + log.Infof("Upgrading %s", itemType) + for _, item := range items { + didUpdate, err := hub.UpgradeItem(itemType, item.Name, force) + if err != nil { + return err + } + if didUpdate { + updated++ + } + } + log.Infof("Upgraded %d %s", updated, itemType) } return nil diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index 924139b93a8..42759aa8fd4 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) +// XXX: this should not need hub? func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index 2b48cdc1231..27a078eb6ca 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -29,14 +29,11 @@ func Suggest(itemType string, baseItem string, suggestItem string, score int, ig } } -func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { +func GetDistance(hub *cwhub.Hub, itemType string, itemName string) (*cwhub.Item, int) { allItems := make([]string, 0) nearestScore := 100 nearestItem := &cwhub.Item{} - // XXX: handle error - hub, _ := cwhub.GetHub() - hubItems := hub.GetItemMap(itemType) for _, item := range hubItems { allItems = append(allItems, item.Name) @@ -53,7 +50,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } @@ -70,7 +67,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string, } func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { return nil, cobra.ShellCompDirectiveDefault } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index df612ab1a89..a4e902afdf8 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -10,7 +10,6 @@ import ( "github.com/crowdsecurity/go-cs-lib/coalesce" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) type cmdHelp struct { @@ -168,19 +167,6 @@ func NewItemsCmd(typeName string) *cobra.Command { Args: cobra.MinimumNArgs(1), Aliases: []string{it.singular}, DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, } cmd.AddCommand(NewItemsInstallCmd(typeName)) @@ -211,29 +197,29 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } - hub, err := cwhub.GetHub() + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) if err != nil { return err } - branch := chooseHubBranch() - for _, name := range args { t := hub.GetItem(it.name, name) if t == nil { - nearestItem, score := GetDistance(it.name, name) + nearestItem, score := GetDistance(hub, it.name, name) Suggest(it.name, name, nearestItem.Name, score, ignoreError) continue } - if err := hub.InstallItem(name, it.name, force, downloadOnly, hubURLTemplate, branch); err != nil { + if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } log.Errorf("Error while installing '%s': %s", name, err) } } + // XXX: only reload if we installed something + log.Infof(ReloadMessage()) return nil } @@ -283,17 +269,32 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e return err } - hub, err := cwhub.GetHub() + hub, err := require.Hub(csConfig, nil) if err != nil { return err } if all { - err := hub.RemoveMany(it.name, "", all, purge, force) + items, err := hub.GetInstalledItems(it.name) if err != nil { return err } + removed := 0 + for _, item := range items { + didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force) + if err != nil { + return err + } + if didRemove { + removed++ + } + } + log.Infof("Removed %d %s", removed, it.name) + if removed > 0 { + log.Infof(ReloadMessage()) + } + return nil } @@ -301,6 +302,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) } + removed := 0 for _, name := range args { if !force { item := hub.GetItem(it.name, name) @@ -315,10 +317,18 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e } } - err := hub.RemoveMany(it.name, name, all, purge, force) + didRemove, err := hub.RemoveItem(it.name, name, purge, force) if err != nil { return err } + + if didRemove { + log.Infof("Removed %s", name) + removed++ + } + } + if removed > 0 { + log.Infof(ReloadMessage()) } return nil @@ -364,17 +374,32 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } - hub, err := cwhub.GetHub() + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) if err != nil { return err } - branch := chooseHubBranch() - if all { - if err := hub.UpgradeConfig(it.name, "", force, hubURLTemplate, branch); err != nil { + items, err := hub.GetInstalledItems(it.name) + if err != nil { return err } + + updated := 0 + for _, item := range items { + didUpdate, err := hub.UpgradeItem(it.name, item.Name, force) + if err != nil { + return err + } + if didUpdate { + updated++ + } + } + log.Infof("Updated %d %s", updated, it.name) + if updated > 0 { + log.Infof(ReloadMessage()) + } + return nil } @@ -382,10 +407,19 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) } + updated := 0 for _, name := range args { - if err := hub.UpgradeConfig(it.name, name, force, hubURLTemplate, branch); err != nil { + didUpdate, err := hub.UpgradeItem(it.name, name, force) + if err != nil { return err } + if didUpdate { + log.Infof("Updated %s", name) + updated++ + } + } + if updated > 0 { + log.Infof(ReloadMessage()) } return nil @@ -434,8 +468,17 @@ func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + for _, name := range args { - if err = InspectItem(name, it.name, noMetrics); err != nil { + item := hub.GetItem(it.name, name) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if err = InspectItem(hub, item, noMetrics); err != nil { return err } } @@ -478,7 +521,12 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err return err } - if err = ListItems(color.Output, []string{it.name}, args, false, true, all); err != nil { + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if err = ListItems(hub, color.Output, []string{it.name}, args, false, true, all); err != nil { return err } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 371cdc16d4f..46c5a0c9e1c 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -5,23 +5,22 @@ import ( "encoding/json" "fmt" "io" - "slices" + "os" "sort" "strings" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" + "slices" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) - func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) { itemNames := hub.GetItemNames(itemType) notExist := []string{} if len(args) > 0 { - installedOnly = false for _, arg := range args { if !slices.Contains(itemNames, arg) { notExist = append(notExist, arg) @@ -35,6 +34,7 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b if len(args) > 0 { itemNames = args + installedOnly = false } if installedOnly { @@ -49,25 +49,19 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b return itemNames, nil } - -func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { +func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { var err error - hub, err := cwhub.GetHub() - if err != nil { - return err - } - items := make(map[string][]string) for _, itemType := range itemTypes { if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil { return err } } - + if csConfig.Cscli.Output == "human" { for _, itemType := range itemTypes { - listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) + listHubItemTable(hub, out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) } } else if csConfig.Cscli.Output == "json" { type itemHubStatus struct { @@ -114,7 +108,6 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, if err != nil { log.Fatalf("failed to write header: %s", err) } - } for _, itemType := range itemTypes { for _, itemName := range items[itemType] { @@ -143,40 +136,34 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, return nil } -func InspectItem(name string, itemType string, noMetrics bool) error { - hub, err := cwhub.GetHub() - if err != nil { - return err - } - - hubItem := hub.GetItem(itemType, name) - if hubItem == nil { - return fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - var b []byte +func InspectItem(hub *cwhub.Hub, item *cwhub.Item, noMetrics bool) error { + var ( + b []byte + err error + ) switch csConfig.Cscli.Output { case "human", "raw": - b, err = yaml.Marshal(*hubItem) + enc := yaml.NewEncoder(os.Stdout) + enc.SetIndent(2) + err = enc.Encode(item) if err != nil { - return fmt.Errorf("unable to marshal item: %s", err) + return fmt.Errorf("unable to encode item: %s", err) } case "json": - b, err = json.MarshalIndent(*hubItem, "", " ") + b, err = json.MarshalIndent(*item, "", " ") if err != nil { return fmt.Errorf("unable to marshal item: %s", err) } + fmt.Printf("%s", string(b)) } - fmt.Printf("%s", string(b)) - if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { return nil } fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hub, hubItem) + ShowMetrics(hub, item) return nil } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index aa43a3d6d83..6cfe40a8607 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -38,7 +38,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { log.Fatalf("parsing api url ('%s'): %s", apiurl, err) } - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { log.Fatal(err) } @@ -339,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs log.Fatalf("Failed to init expr helpers : %s", err) } - _, err = require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { log.Fatal(err) } - csParsers := parser.NewParsers() + csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { log.Fatalf("unable to load parsers: %s", err) } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 8f5561d3d2c..da75ed439db 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -30,6 +30,9 @@ var OutputColor string var mergedConfig string +// flagBranch overrides the value in csConfig.Cscli.HubBranch +var flagBranch = "" + func initConfig() { var err error if trace_lvl { @@ -65,8 +68,8 @@ func initConfig() { log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) } - if hubBranch == "" && csConfig.Cscli.HubBranch != "" { - hubBranch = csConfig.Cscli.HubBranch + if flagBranch != "" { + csConfig.Cscli.HubBranch = flagBranch } if OutputFormat != "" { @@ -197,7 +200,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error") rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace") - rootCmd.PersistentFlags().StringVar(&hubBranch, "branch", "", "Override hub branch on github") + rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github") if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { log.Fatalf("failed to hide flag: %s", err) } diff --git a/cmd/crowdsec-cli/require/branch.go b/cmd/crowdsec-cli/require/branch.go new file mode 100644 index 00000000000..14d48279652 --- /dev/null +++ b/cmd/crowdsec-cli/require/branch.go @@ -0,0 +1,61 @@ +package require + +// Set the appropriate hub branch according to config settings and crowdsec version + +import ( + log "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" +) + +func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string { + if cfg.Cscli.HubBranch != "" { + logger.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch) + return cfg.Cscli.HubBranch + } + + latest, err := cwversion.Latest() + if err != nil { + logger.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err) + return "master" + } + + csVersion := cwversion.VersionStrip() + if csVersion == latest { + logger.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion) + return "master" + } + + // if current version is greater than the latest we are in pre-release + if semver.Compare(csVersion, latest) == 1 { + logger.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion) + return "master" + } + + if csVersion == "" { + logger.Warning("Crowdsec version is not set, using hub branch 'master'") + return "master" + } + + log.Warnf("A new CrowdSec release is available (%s). "+ + "Your version is '%s'. Please update it to use new parsers/scenarios/collections.", + latest, csVersion) + return csVersion +} + + +// HubBranch sets the branch (in cscli config) and returns its value +// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag +func HubBranch(cfg *csconfig.Config) string { + // XXX: we want to be able to suppress logs + // to avoid being too noisy in some commands + logger := log.StandardLogger() + + branch := chooseBranch(cfg, logger) + + cfg.Cscli.HubBranch = branch + + return branch +} diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 2985a06cf2b..d87decc5679 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -64,12 +64,29 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub(c *csconfig.Config) (*cwhub.Hub, error) { - if c.Hub == nil { +// RemoteHub returns the configuration required to download hub index and items: url, branch, etc. +func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg { + // set branch in config, and log if necessary + branch := HubBranch(c) + remote := &cwhub.RemoteHubCfg { + Branch: branch, + URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", + IndexPath: ".index.json", + } + + return remote +} + +// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items. +// If no remote parameter is provided, the hub can only be used for local operations. +func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) { + local := c.Hub + + if local == nil { return nil, fmt.Errorf("you must configure cli before interacting with hub") } - hub, err := cwhub.InitHub(c.Hub) + hub, err := cwhub.InitHub(local, remote) if err != nil { return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) } diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go index f589951e0ec..884aa989036 100644 --- a/cmd/crowdsec-cli/setup.go +++ b/cmd/crowdsec-cli/setup.go @@ -13,6 +13,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/setup" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) // NewSetupCmd defines the "cscli setup" command. @@ -298,14 +300,17 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error { return err } - branch := chooseHubBranch() - input, err := os.ReadFile(fromFile) if err != nil { return fmt.Errorf("while reading file %s: %w", fromFile, err) } - if err = setup.InstallHubItems(csConfig, input, dryRun, hubURLTemplate, branch); err != nil { + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + if err = setup.InstallHubItems(hub, input, dryRun); err != nil { return err } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 0ad4ff5aedd..83f01bc15a9 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -145,7 +145,7 @@ func NewSimulationEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { log.Fatal(err) } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index e8462c4ae11..76043d2ee72 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -128,10 +128,10 @@ func collectOSInfo() ([]byte, error) { return w.Bytes(), nil } -func collectHubItems(itemType string) []byte { +func collectHubItems(hub *cwhub.Hub, itemType string) []byte { out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil { + if err := ListItems(hub, out, []string{itemType}, []string{}, false, true, false); err != nil { log.Warnf("could not collect %s list: %s", itemType, err) } return out.Bytes() @@ -293,7 +293,7 @@ cscli support dump -f /tmp/crowdsec-support.zip skipAgent = true } - hub, err := require.Hub(csConfig) + hub, err := require.Hub(csConfig, nil) if err != nil { log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") skipHub = true @@ -332,10 +332,10 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig() if !skipHub { - infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) - infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) - infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS) - infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) + infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS) + infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS) + infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS) + infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS) } if !skipDB { diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 1f3f62038ac..0336f40a1ee 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -3,6 +3,7 @@ package main import ( "fmt" "io" + "strconv" "github.com/aquasecurity/table" "github.com/enescakir/emoji" @@ -10,14 +11,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) { +func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, itemNames []string) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - hub, _ := cwhub.GetHub() - for itemName := range itemNames { item := hub.GetItem(itemType, itemNames[itemName]) status, emo := item.Status() @@ -35,11 +34,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired") t.AddRow( - fmt.Sprintf("%d", metrics["curr_count"]), - fmt.Sprintf("%d", metrics["overflow"]), - fmt.Sprintf("%d", metrics["instantiation"]), - fmt.Sprintf("%d", metrics["pour"]), - fmt.Sprintf("%d", metrics["underflow"]), + strconv.Itoa(metrics["curr_count"]), + strconv.Itoa(metrics["overflow"]), + strconv.Itoa(metrics["instantiation"]), + strconv.Itoa(metrics["pour"]), + strconv.Itoa(metrics["underflow"]), ) renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName)) @@ -47,23 +46,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int } func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) { - skip := true t := newTable(out) t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed") + // don't show table if no hits + showTable := false + for source, stats := range metrics { if stats["hits"] > 0 { t.AddRow( source, - fmt.Sprintf("%d", stats["hits"]), - fmt.Sprintf("%d", stats["parsed"]), - fmt.Sprintf("%d", stats["unparsed"]), + strconv.Itoa(stats["hits"]), + strconv.Itoa(stats["parsed"]), + strconv.Itoa(stats["unparsed"]), ) - skip = false + showTable = true } } - if !skip { + if showTable { renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName)) t.Render() } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index dd81ad34564..1b688f2d53e 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,13 +23,13 @@ import ( func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error - hub, err := cwhub.InitHub(cConfig.Hub) + hub, err := cwhub.InitHub(cConfig.Hub, nil) if err != nil { return nil, fmt.Errorf("while loading hub index: %w", err) } // Start loading configs - csParsers := parser.NewParsers() + csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { return nil, fmt.Errorf("while loading parsers: %w", err) } diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 27bb32b419a..1333e12031c 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -36,7 +36,7 @@ type Config struct { PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"` DisableAPI bool `yaml:"-"` DisableAgent bool `yaml:"-"` - Hub *HubCfg `yaml:"-"` + Hub *LocalHubCfg `yaml:"-"` } func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { diff --git a/pkg/csconfig/config_test.go b/pkg/csconfig/config_test.go index 9bdf2da6d08..4843c2f70f9 100644 --- a/pkg/csconfig/config_test.go +++ b/pkg/csconfig/config_test.go @@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) { require.NoError(t, err) _, _, err = NewConfig("./testdata/xxx.yaml", false, false, false) - assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage) + require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage) _, _, err = NewConfig("./testdata/simulation.yaml", false, false, false) - assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config") + require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config") } func TestNewCrowdSecConfig(t *testing.T) { diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index d3638c8885d..ca3750e5812 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,7 +1,7 @@ package csconfig -// HubCfg holds the configuration for a hub -type HubCfg struct { +// LocalHubCfg holds the configuration for a local hub: where to download etc. +type LocalHubCfg struct { HubIndexFile string // Path to the local index file HubDir string // Where the hub items are downloaded InstallDir string // Where to install items @@ -9,7 +9,7 @@ type HubCfg struct { } func (c *Config) loadHub() error { - c.Hub = &HubCfg{ + c.Hub = &LocalHubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, InstallDir: c.ConfigPaths.ConfigDir, diff --git a/pkg/csconfig/hub_test.go b/pkg/csconfig/hub_test.go index 0fa627ae0c1..2f9528c6043 100644 --- a/pkg/csconfig/hub_test.go +++ b/pkg/csconfig/hub_test.go @@ -12,7 +12,7 @@ func TestLoadHub(t *testing.T) { tests := []struct { name string input *Config - expected *HubCfg + expected *LocalHubCfg expectedErr string }{ { @@ -25,7 +25,7 @@ func TestLoadHub(t *testing.T) { HubIndexFile: "./hub/.index.json", }, }, - expected: &HubCfg{ + expected: &LocalHubCfg{ HubDir: "./hub", HubIndexFile: "./hub/.index.json", InstallDir: "./testdata", diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 45e2852bd61..1b56bfebd8f 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -32,23 +32,23 @@ func testHub(t *testing.T, update bool) *Hub { tmpDir, err := os.MkdirTemp("", "testhub") require.NoError(t, err) - hubCfg := &csconfig.HubCfg{ + local := &csconfig.LocalHubCfg{ HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"), InstallDir: filepath.Join(tmpDir, "crowdsec"), InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(hubCfg.HubDir, 0o700) + err = os.MkdirAll(local.HubDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDir, 0o700) + err = os.MkdirAll(local.InstallDir, 0o700) require.NoError(t, err) - err = os.MkdirAll(hubCfg.InstallDataDir, 0o700) + err = os.MkdirAll(local.InstallDataDir, 0o700) require.NoError(t, err) - index, err := os.Create(hubCfg.HubIndexFile) + index, err := os.Create(local.HubIndexFile) require.NoError(t, err) _, err = index.WriteString(`{}`) @@ -62,11 +62,17 @@ func testHub(t *testing.T, update bool) *Hub { var hub *Hub + remote := &RemoteHubCfg{ + Branch: "master", + URLTemplate: mockURLTemplate, + IndexPath: ".index.json", + } + if update { - hub, err = InitHubUpdate(hubCfg, mockURLTemplate, "master", ".index.json") + hub, err = InitHubUpdate(local, remote) require.NoError(t, err) } else { - hub, err = InitHub(hubCfg) + hub, err = InitHub(local, remote) require.NoError(t, err) } diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index a072fbb1c2d..f23f4878285 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -6,6 +6,7 @@ import ( "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDownloadFile(t *testing.T) { @@ -29,21 +30,21 @@ func TestDownloadFile(t *testing.T) { ) err := downloadFile("https://example.com/xx", examplePath) - assert.NoError(t, err) + require.NoError(t, err) content, err := os.ReadFile(examplePath) assert.Equal(t, "example content oneoneone", string(content)) - assert.NoError(t, err) + require.NoError(t, err) //bad uri err = downloadFile("https://zz.com", examplePath) - assert.Error(t, err) + require.Error(t, err) //404 err = downloadFile("https://example.com/x", examplePath) - assert.Error(t, err) + require.Error(t, err) //bad target err = downloadFile("https://example.com/xx", "") - assert.Error(t, err) + require.Error(t, err) } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 4aac75927b1..0e60212adb8 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -16,7 +16,7 @@ import ( func (h *Hub) EnableItem(target *Item) error { var err error - parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") + parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") // create directories if needed if target.Installed { @@ -24,7 +24,7 @@ func (h *Hub) EnableItem(target *Item) error { return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) } - if target.Local { + if target.IsLocal() { return fmt.Errorf("%s is local, won't enable", target.Name) } @@ -65,7 +65,7 @@ func (h *Hub) EnableItem(target *Item) error { } // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) + srcPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while getting source path: %w", err) } @@ -87,7 +87,7 @@ func (h *Hub) EnableItem(target *Item) error { } func (h *Hub) purgeItem(target Item) (Item, error) { - itempath := h.cfg.HubDir + "/" + target.RemotePath + itempath := h.local.HubDir + "/" + target.RemotePath // disable hub file if err := os.Remove(itempath); err != nil { @@ -117,7 +117,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return nil } - if target.Local { + if target.IsLocal() { return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) } @@ -128,11 +128,11 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { // for a COLLECTIONS, disable sub-items if target.Type == COLLECTIONS { for _, sub := range target.SubItems() { - val, ok := h.Items[sub.Type][sub.Name] + val, ok := h.Items[sub.Type][sub.Name] if !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) continue - } + } // check if the item doesn't belong to another collection before removing it toRemove := true @@ -155,7 +155,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { } } - syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) + syml, err := filepath.Abs(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) if err != nil { return err } @@ -178,7 +178,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return fmt.Errorf("while reading symlink: %w", err) } - absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) + absPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while abs path: %w", err) } diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index a5d731fee7b..864b3fa8587 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -10,7 +10,7 @@ import ( func testInstall(hub *Hub, t *testing.T, item Item) { // Install the parser - err := hub.DownloadLatest(&item, false, false, mockURLTemplate, "master") + err := hub.DownloadLatest(&item, false, false) require.NoError(t, err, "failed to download %s", item.Name) _, err = hub.LocalSync() @@ -51,7 +51,7 @@ func testUpdate(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := hub.DownloadLatest(&item, true, true, mockURLTemplate, "master") + err := hub.DownloadLatest(&item, true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 6f42bed65e3..aae2f23dd1e 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -7,6 +7,7 @@ package cwhub import ( "bytes" "crypto/sha256" + "encoding/hex" "fmt" "io" "net/http" @@ -19,7 +20,7 @@ import ( ) // InstallItem installs an item from the hub -func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool, hubURLTemplate, branch string) error { +func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) @@ -33,7 +34,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } } - err := h.DownloadLatest(item, force, true, hubURLTemplate, branch) + err := h.DownloadLatest(item, force, true) if err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } @@ -43,7 +44,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } if downloadOnly { - log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath)) + log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath)) return nil } @@ -61,131 +62,97 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly return nil } -// RemoveItem removes one - or all - the items from the hub -func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error { - if name != "" { - item := h.GetItem(itemType, name) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - err := h.DisableItem(item, purge, forceAction) - - if err != nil { - return fmt.Errorf("unable to disable %s: %w", item.Name, err) - } - - if err = h.AddItem(*item); err != nil { - return fmt.Errorf("unable to add %s: %w", item.Name, err) - } +// RemoveItem disables one item, optionally removing the downloaded content +func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction bool) (bool, error) { + removed := false - return nil + item := h.GetItem(itemType, name) + if item == nil { + return false, fmt.Errorf("can't find '%s' in %s", name, itemType) } - if !all { - return fmt.Errorf("removing item: no item specified") + if !item.Downloaded { + log.Infof("removing %s: not downloaded -- no removal required", item.Name) + return false, nil } - disabled := 0 + if !item.Installed && !purge { + log.Infof("removing %s: already uninstalled", item.Name) + return false, nil + } - // remove all - for _, v := range h.GetItemMap(itemType) { - if !v.Installed { - continue - } + if err := h.DisableItem(item, purge, forceAction); err != nil { + return false, fmt.Errorf("unable to disable %s: %w", item.Name, err) + } - err := h.DisableItem(&v, purge, forceAction) - if err != nil { - return fmt.Errorf("unable to disable %s: %w", v.Name, err) - } + // XXX: should take the value from DisableItem + removed = true - if err := h.AddItem(v); err != nil { - return fmt.Errorf("unable to add %s: %w", v.Name, err) - } - disabled++ + if err := h.AddItem(*item); err != nil { + return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err) } - log.Infof("Disabled %d items", disabled) - - return nil + return removed, nil } -// UpgradeConfig upgrades an item from the hub -func (h *Hub) UpgradeConfig(itemType string, name string, force bool, hubURLTemplate, branch string) error { - updated := 0 - found := false - - for _, v := range h.GetItemMap(itemType) { - if name != "" && name != v.Name { - continue - } +// UpgradeItem upgrades an item from the hub +func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error) { + updated := false - if !v.Installed { - log.Tracef("skip %s, not installed", v.Name) - continue - } - - if !v.Downloaded { - log.Warningf("%s: not downloaded, please install.", v.Name) - continue - } + item := h.GetItem(itemType, name) + if item == nil { + return false, fmt.Errorf("can't find '%s' in %s", name, itemType) + } - found = true + if !item.Downloaded { + return false, fmt.Errorf("can't upgrade %s: not installed", item.Name) + } - if v.UpToDate { - log.Infof("%s: up-to-date", v.Name) + if !item.Installed { + return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", item.Name) + } - if err := h.DownloadDataIfNeeded(v, force); err != nil { - return fmt.Errorf("%s: download failed: %w", v.Name, err) - } + if item.UpToDate { + log.Infof("%s: up-to-date", item.Name) - if !force { - continue - } + if err := h.DownloadDataIfNeeded(*item, force); err != nil { + return false, fmt.Errorf("%s: download failed: %w", item.Name, err) } - if err := h.DownloadLatest(&v, force, true, hubURLTemplate, branch); err != nil { - return fmt.Errorf("%s: download failed: %w", v.Name, err) + if !force { + // no upgrade needed + return false, nil } + } - if !v.UpToDate { - if v.Tainted { - log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name) - } else if v.Local { - log.Infof("%v %s is local", emoji.Prohibited, v.Name) - } - } else { - // this is used while scripting to know if the hub has been upgraded - // and a configuration reload is required - fmt.Printf("updated %s\n", v.Name) - log.Infof("%v %s : updated", emoji.Package, v.Name) - updated++ - } + if err := h.DownloadLatest(item, force, true); err != nil { + return false, fmt.Errorf("%s: download failed: %w", item.Name, err) + } - if err := h.AddItem(v); err != nil { - return fmt.Errorf("unable to add %s: %w", v.Name, err) + if !item.UpToDate { + if item.Tainted { + log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, item.Name) + } else if item.IsLocal() { + log.Infof("%v %s is local", emoji.Prohibited, item.Name) } + } else { + // a check on stdout is used while scripting to know if the hub has been upgraded + // and a configuration reload is required + // TODO: use a better way to communicate this + fmt.Printf("updated %s\n", item.Name) + log.Infof("%v %s: updated", emoji.Package, item.Name) + updated = true } - if !found && name == "" { - log.Infof("No %s installed, nothing to upgrade", itemType) - } else if !found { - log.Errorf("can't find '%s' in %s", name, itemType) - } else if updated == 0 && found { - if name == "" { - log.Infof("All %s are already up-to-date", itemType) - } else { - log.Infof("Item '%s' is up-to-date", name) - } - } else if updated != 0 { - log.Infof("Upgraded %d items", updated) + if err := h.AddItem(*item); err != nil { + return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err) } - return nil + return updated, nil } // DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubURLTemplate, branch string) error { +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -196,7 +163,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU return nil } - return h.DownloadItem(target, overwrite, hubURLTemplate, branch) + return h.DownloadItem(target, overwrite) } // collection @@ -216,7 +183,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU if sub.Type == COLLECTIONS { log.Tracef("collection, recurse") - err = h.DownloadLatest(&val, overwrite, updateOnly, hubURLTemplate, branch) + err = h.DownloadLatest(&val, overwrite, updateOnly) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -224,7 +191,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU downloaded := val.Downloaded - err = h.DownloadItem(&val, overwrite, hubURLTemplate, branch) + err = h.DownloadItem(&val, overwrite) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -240,7 +207,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU h.Items[sub.Type][sub.Name] = val } - err = h.DownloadItem(target, overwrite, hubURLTemplate, branch) + err = h.DownloadItem(target, overwrite) if err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -248,9 +215,13 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU return nil } -func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch string) error { - url := fmt.Sprintf(hubURLTemplate, branch, target.RemotePath) - tdir := h.cfg.HubDir +func (h *Hub) DownloadItem(target *Item, overwrite bool) error { + url, err := h.remote.urlTo(target.RemotePath) + if err != nil { + return fmt.Errorf("failed to build hub item request: %w", err) + } + + tdir := h.local.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { @@ -291,7 +262,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch return fmt.Errorf("while hashing %s: %w", target.Name, err) } - meow := fmt.Sprintf("%x", hash.Sum(nil)) + meow := hex.EncodeToString(hash.Sum(nil)) if meow != target.Versions[target.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) @@ -347,7 +318,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch target.Tainted = false target.UpToDate = true - if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + if err = downloadData(h.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } @@ -358,7 +329,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch // DownloadDataIfNeeded downloads the data files for an item func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.local.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) if err != nil { @@ -367,7 +338,7 @@ func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { defer itemFile.Close() - if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { + if err = downloadData(h.local.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 234e54caa83..f6ea249545a 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -10,14 +10,14 @@ import ( // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed -func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { +func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub := envSetup(t) // fresh install of collection require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -33,18 +33,25 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - hub, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + hub, err := InitHubUpdate(hub.local, remote) require.NoError(t, err, "failed to download index: %s", err) - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") + didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) + require.True(t, didUpdate) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) @@ -53,7 +60,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. -func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { +func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) // fresh install of collection @@ -61,7 +68,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -70,10 +77,17 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) require.NoError(t, err) + require.True(t, didRemove) + + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) @@ -81,19 +95,20 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") + hub, err = InitHubUpdate(hub.local, remote) require.NoError(t, err, "failed to download index: %s", err) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") + didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) + require.False(t, didUpdate) - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test -func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { - hub, err := InitHub(hubCfg) +func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { + hub, err := InitHub(local, remote) require.NoError(t, err, "failed to load hub index") return hub @@ -102,7 +117,7 @@ func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. -func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { +func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { hub := envSetup(t) // fresh install of collection @@ -110,7 +125,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -119,10 +134,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) require.NoError(t, err) + require.True(t, didRemove) + + remote := &RemoteHubCfg{ + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails @@ -136,16 +158,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") + hub, err = InitHubUpdate(hub.local, remote) require.NoError(t, err, "failed to download index: %s", err) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") + didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) + require.True(t, didUpdate) - hub = getHubOrFail(t, hub.cfg) + hub = getHubOrFail(t, hub.local, remote) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 3adf1c308ca..d695f0b8b7b 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -17,10 +17,17 @@ import ( // const HubIndexFile = ".index.json" -// Hub represents the runtime status of the hub (parsed items, etc.) +// RemoteHubCfg contains where to find the remote hub, which branch etc. +type RemoteHubCfg struct { + Branch string + URLTemplate string + IndexPath string +} + type Hub struct { Items HubItems - cfg *csconfig.HubCfg + local *csconfig.LocalHubCfg + remote *RemoteHubCfg skippedLocal int skippedTainted int } @@ -32,6 +39,7 @@ var ( // GetHub returns the hub singleton // it returns an error if it's not initialized to avoid nil dereference +// XXX: convenience function that we should get rid of at some point func GetHub() (*Hub, error) { if theHub == nil { return nil, fmt.Errorf("hub not initialized") @@ -41,14 +49,14 @@ func GetHub() (*Hub, error) { } // InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use -func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { - if cfg == nil { - return nil, fmt.Errorf("no configuration found for hub") +func InitHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { + if local == nil { + return nil, fmt.Errorf("no hub configuration found") } - log.Debugf("loading hub idx %s", cfg.HubIndexFile) + log.Debugf("loading hub idx %s", local.HubIndexFile) - bidx, err := os.ReadFile(cfg.HubIndexFile) + bidx, err := os.ReadFile(local.HubIndexFile) if err != nil { return nil, fmt.Errorf("unable to read index file: %w", err) } @@ -64,8 +72,9 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { } theHub = &Hub{ - Items: ret, - cfg: cfg, + Items: ret, + local: local, + remote: remote, } _, err = theHub.LocalSync() @@ -78,12 +87,12 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { // InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk // It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) (*Hub, error) { - if cfg == nil { +func InitHubUpdate(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { + if local == nil { return nil, fmt.Errorf("no configuration found for hub") } - bidx, err := DownloadIndex(cfg.HubIndexFile, urlTemplate, branch, remotePath) + bidx, err := remote.DownloadIndex(local.HubIndexFile) if err != nil { return nil, fmt.Errorf("failed to download index: %w", err) } @@ -96,8 +105,9 @@ func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) } theHub = &Hub{ - Items: ret, - cfg: cfg, + Items: ret, + local: local, + remote: remote, } if _, err := theHub.LocalSync(); err != nil { @@ -107,10 +117,20 @@ func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) return theHub, nil } +func (r RemoteHubCfg) urlTo(remotePath string) (string, error) { + if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { + return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) + } + + return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil +} + // DownloadIndex downloads the latest version of the index and returns the content -func DownloadIndex(localPath, hubURLTemplate, branch, remotePath string) ([]byte, error) { - url := fmt.Sprintf(hubURLTemplate, branch, remotePath) - log.Debugf("fetching index from branch %s (%s)", branch, url) +func (r RemoteHubCfg) DownloadIndex(localPath string) ([]byte, error) { + url, err := r.urlTo(r.IndexPath) + if err != nil { + return nil, fmt.Errorf("failed to build hub index request: %w", err) + } req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -212,14 +232,17 @@ func ParseIndex(buff []byte) (HubItems, error) { // ItemStats returns total counts of the hub items func (h *Hub) ItemStats() []string { loaded := "" + for _, itemType := range ItemTypes { // ensure the order is always the same if h.Items[itemType] == nil { continue } + if len(h.Items[itemType]) == 0 { continue } + loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) } diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 02c552c4c0c..b4b0cd7d4f5 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,7 +13,13 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - _, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") + remote := &RemoteHubCfg { + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + _, err := InitHubUpdate(hub.local, remote) require.NoError(t, err) _, err = GetHub() @@ -31,23 +37,44 @@ func TestDownloadIndex(t *testing.T) { os.Remove(tmpIndex.Name()) }) - ret, err := DownloadIndex(tmpIndex.Name(), "x", "", "") - cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ") + hub := envSetup(t) + + hub.remote = &RemoteHubCfg { + URLTemplate: "x", + Branch: "", + IndexPath: "", + } + + ret, err := hub.remote.DownloadIndex(tmpIndex.Name()) + cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") fmt.Printf("->%+v", ret) // bad domain fmt.Println("Test 'bad domain'") - ret, err = DownloadIndex(tmpIndex.Name(), "https://baddomain/%s/%s", "master", ".index.json") - cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") + hub.remote = &RemoteHubCfg { + URLTemplate: "https://baddomain/%s/%s", + Branch: "master", + IndexPath: ".index.json", + } + + ret, err = hub.remote.DownloadIndex(tmpIndex.Name()) +// XXX: this is not failing +// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") fmt.Printf("->%+v", ret) // bad target path fmt.Println("Test 'bad target path'") - ret, err = DownloadIndex("/does/not/exist/index.json", mockURLTemplate, "master", ".index.json") + hub.remote = &RemoteHubCfg { + URLTemplate: mockURLTemplate, + Branch: "master", + IndexPath: ".index.json", + } + + ret, err = hub.remote.DownloadIndex("/does/not/exist/index.json") cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") fmt.Printf("->%+v", ret) diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 7d20b107e12..ff890d84ef3 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -1,6 +1,7 @@ package cwhub import ( + "encoding/json" "fmt" "github.com/enescakir/emoji" @@ -49,11 +50,10 @@ type Item struct { LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} LocalVersion string `json:"local_version,omitempty"` LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed,omitempty"` + Installed bool `json:"installed,omitempty"` // XXX: should we remove omitempty from bool fields? Downloaded bool `json:"downloaded,omitempty"` UpToDate bool `json:"up_to_date,omitempty"` Tainted bool `json:"tainted,omitempty"` // has it been locally modified? - Local bool `json:"local,omitempty"` // if it's a non versioned control one // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` @@ -62,35 +62,73 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } - type SubItem struct { Type string Name string } +func (i *Item) IsLocal() bool { + return i.Installed && !i.Downloaded +} + +// MarshalJSON is used to add the "local" field to the json output +// (i.e. with cscli ... inspect -o json) +// It must not use a pointer receiver +func (i Item) MarshalJSON() ([]byte, error) { + type Alias Item + return json.Marshal(&struct { + Alias + Local bool `json:"local"` // XXX: omitempty? + }{ + Alias: Alias(i), + Local: i.IsLocal(), + }) +} + +// MarshalYAML is used to add the "local" field to the yaml output +// (i.e. with cscli ... inspect -o raw) +// It must not use a pointer receiver +func (i Item) MarshalYAML() (interface{}, error) { + type Alias Item + return &struct { + Alias `yaml:",inline"` + Local bool `yaml:"local"` + }{ + Alias: Alias(i), + Local: i.IsLocal(), + }, nil +} + +// SubItems returns the list of sub items for a given item (typically a collection) func (i *Item) SubItems() []SubItem { sub := make([]SubItem, - len(i.Parsers) + - len(i.PostOverflows) + - len(i.Scenarios) + - len(i.Collections)) + len(i.Parsers)+ + len(i.PostOverflows)+ + len(i.Scenarios)+ + len(i.Collections)) + n := 0 + for _, name := range i.Parsers { sub[n] = SubItem{Type: PARSERS, Name: name} n++ } + for _, name := range i.PostOverflows { sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name} n++ } + for _, name := range i.Scenarios { sub[n] = SubItem{Type: SCENARIOS, Name: name} n++ } + for _, name := range i.Collections { sub[n] = SubItem{Type: COLLECTIONS, Name: name} n++ } + return sub } @@ -106,7 +144,7 @@ func (i *Item) Status() (string, emoji.Emoji) { } managed := true - if i.Local { + if i.IsLocal() { managed = false status += ",local" } @@ -115,7 +153,7 @@ func (i *Item) Status() (string, emoji.Emoji) { if i.Tainted { warning = true status += ",tainted" - } else if !i.UpToDate && !i.Local { + } else if !i.UpToDate && !i.IsLocal() { warning = true status += ",update-available" } @@ -201,7 +239,7 @@ func (h *Hub) AddItem(item Item) error { func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { items, ok := h.Items[itemType] if !ok { - return nil, fmt.Errorf("no %s in hubIdx", itemType) + return nil, fmt.Errorf("no %s in the hub index", itemType) } retItems := make([]Item, 0) diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index e16dbe31200..9971771e97b 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -22,19 +22,19 @@ func TestItemStatus(t *testing.T) { item.Installed = true item.UpToDate = false - item.Local = false item.Tainted = false + item.Downloaded = true txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) - item.Installed = false + item.Installed = true item.UpToDate = false - item.Local = true item.Tainted = false + item.Downloaded = false txt, _ = item.Status() - require.Equal(t, "disabled,local", txt) + require.Equal(t, "enabled,local", txt) } stats := hub.ItemStats() diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go index fe26aa4c107..e57d6fa5a26 100644 --- a/pkg/cwhub/leakybucket.go +++ b/pkg/cwhub/leakybucket.go @@ -37,13 +37,15 @@ func itemKey(itemPath string) (string, error) { return fmt.Sprintf("%s/%s", author, fname), nil } -// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. +// GetItemByPath retrieves the item from the hub index based on its path. +// To achieve this it resolves a symlink to find the associated hub item. func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { itemKey, err := itemKey(itemPath) if err != nil { return nil, err } + // XXX: use GetItem() m := h.GetItemMap(itemType) if m == nil { return nil, fmt.Errorf("item type %s doesn't exist", itemType) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 24f948d3c26..7dd04ff2c0f 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -2,6 +2,7 @@ package cwhub import ( "crypto/sha256" + "encoding/hex" "fmt" "io" "os" @@ -50,7 +51,7 @@ func getSHA256(filepath string) (string, error) { return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) } - return fmt.Sprintf("%x", h.Sum(nil)), nil + return hex.EncodeToString(h.Sum(nil)), nil } type itemFileInfo struct { @@ -64,8 +65,8 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { ret := itemFileInfo{} inhub := false - hubDir := h.cfg.HubDir - installDir := h.cfg.InstallDir + hubDir := h.local.HubDir + installDir := h.local.InstallDir subs := strings.Split(path, string(os.PathSeparator)) @@ -190,7 +191,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { Stage: info.stage, Installed: true, Type: info.ftype, - Local: true, LocalPath: path, UpToDate: true, FileName: fileName, @@ -229,7 +229,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { continue } - if path == h.cfg.HubDir+"/"+item.RemotePath { + if path == h.local.HubDir+"/"+item.RemotePath { log.Tracef("marking %s as downloaded", item.Name) item.Downloaded = true } @@ -391,11 +391,11 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { for _, scan := range ItemTypes { cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { - log.Errorf("failed %s : %s", cpath, err) + log.Errorf("failed %s: %s", cpath, err) } // explicit check for non existing directory, avoid spamming log.Debug - if _, err := os.Stat(cpath); os.IsNotExist(err) { + if _, err = os.Stat(cpath); os.IsNotExist(err) { log.Tracef("directory %s doesn't exist, skipping", cpath) continue } @@ -435,14 +435,14 @@ func (h *Hub) LocalSync() ([]string, error) { h.skippedLocal = 0 h.skippedTainted = 0 - warnings, err := h.SyncDir(h.cfg.InstallDir) + warnings, err := h.SyncDir(h.local.InstallDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) } - _, err = h.SyncDir(h.cfg.HubDir) + _, err = h.SyncDir(h.local.HubDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) } return warnings, nil diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index f2b309bb54c..e9ffebe0a41 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -39,7 +39,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string - RuntimeHubConfig *csconfig.HubCfg + RuntimeHubConfig *csconfig.LocalHubCfg ResultsPath string ParserResultFile string @@ -117,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName), - RuntimeHubConfig: &csconfig.HubCfg{ + RuntimeHubConfig: &csconfig.LocalHubCfg{ HubDir: runtimeHubFolder, HubIndexFile: hubTest.HubIndexFile, InstallDir: runtimeFolder, @@ -391,7 +391,7 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - hub, err := cwhub.InitHub(t.RuntimeHubConfig) + hub, err := cwhub.InitHub(t.RuntimeHubConfig, nil) if err != nil { log.Fatal(err) } @@ -451,51 +451,51 @@ func (t *HubTestItem) Run() error { } // create runtime folder - if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) } // create runtime data folder - if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) } // create runtime hub folder - if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) } - if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { + if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) } // create results folder - if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { + if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) } // copy template config file to runtime folder - if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { + if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) } // copy template profile file to runtime folder - if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { + if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) } // copy template simulation file to runtime folder - if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { + if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } // copy template patterns folder to runtime folder - if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { + if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) } // install the hub in the runtime folder - if err := t.InstallHub(); err != nil { + if err = t.InstallHub(); err != nil { return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) } @@ -503,7 +503,7 @@ func (t *HubTestItem) Run() error { logType := t.Config.LogType dsn := fmt.Sprintf("file://%s", logFile) - if err := os.Chdir(testPath); err != nil { + if err = os.Chdir(testPath); err != nil { return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err) } diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index 41d463c489a..7e3f3e98817 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -38,12 +38,12 @@ func TestBucket(t *testing.T) { testdata := "./tests" - hubCfg := &csconfig.HubCfg{ + hubCfg := &csconfig.LocalHubCfg{ HubDir: filepath.Join(testdata, "hub"), HubIndexFile: filepath.Join(testdata, "hub", "index.json"), } - _, err := cwhub.InitHub(hubCfg) + _, err := cwhub.InitHub(hubCfg, nil) if err != nil { t.Fatalf("failed to init hub : %s", err) } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 4c2ae9fde57..02dfb435a9e 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -57,7 +57,7 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) { // Return new parsers // nodes and povfwnodes are already initialized in parser.LoadStages -func NewParsers() *Parsers { +func NewParsers(hub *cwhub.Hub) *Parsers { parsers := &Parsers{ Ctx: &UnixParserCtx{}, Povfwctx: &UnixParserCtx{}, @@ -65,9 +65,6 @@ func NewParsers() *Parsers { PovfwStageFiles: make([]Stagefile, 0), } - // XXX: handle error - hub, _ := cwhub.GetHub() - for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { for _, hubParserItem := range hub.GetItemMap(itemType) { if hubParserItem.Installed { diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 0c38db361c3..50d16b52318 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -10,7 +10,6 @@ import ( goccyyaml "github.com/goccy/go-yaml" "gopkg.in/yaml.v3" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -46,17 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubURLTemplate, branch string) error { +func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err } - hub, err := cwhub.InitHub(csConfig.Hub) - if err != nil { - return fmt.Errorf("getting hub index: %w", err) - } - for _, setupItem := range setupEnvelope.Setup { forceAction := false downloadOnly := false @@ -74,7 +68,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR continue } - if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { + if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing collection %s: %w", collection, err) } } @@ -88,7 +82,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR continue } - if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { + if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing parser %s: %w", parser, err) } } @@ -102,7 +96,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR continue } - if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { + if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing scenario %s: %w", scenario, err) } } @@ -116,7 +110,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR continue } - if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { + if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } } diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 7bf27670b02..40dae750ea1 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -57,12 +57,12 @@ teardown() { @test "crowdsec - default logging configuration (empty/missing common section)" { config_set '.common={}' - rune -124 timeout 1s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" refute_output assert_stderr --partial "Starting processing data" config_set 'del(.common)' - rune -124 timeout 1s "${CROWDSEC}" + rune -124 timeout 2s "${CROWDSEC}" refute_output assert_stderr --partial "Starting processing data" } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats new file mode 100644 index 00000000000..c0f10572bb4 --- /dev/null +++ b/test/bats/20_hub.bats @@ -0,0 +1,78 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_purge_all + hub_strip_index +} + +teardown() { + : +} + +#---------- + +@test "cscli hub list" { + # no items + rune -0 cscli hub list + assert_output --regexp ".*COLLECTIONS.*PARSERS.*SCENARIOS.*POSTOVERFLOWS.*" + rune -0 cscli hub list -o json + assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' + rune -0 cscli hub list -o raw + refute_output + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli scenarios install crowdsecurity/telnet-bf + rune -0 cscli hub list + assert_output --regexp ".*COLLECTIONS.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*" + rune -0 cscli hub list -o json + rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) + rune -0 cscli hub list -o raw + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'crowdsecurity/telnet-bf' + refute_output --partial 'crowdsecurity/linux' + + # all items + rune -0 cscli hub list -a + assert_output --regexp ".*COLLECTIONS.*crowdsecurity/linux.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*" + rune -0 cscli hub list -a -o json + rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) + rune -0 cscli hub list -a -o raw + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'crowdsecurity/telnet-bf' + assert_output --partial 'crowdsecurity/linux' +} + +@test "cscli hub update" { + #XXX: todo + : +} + +@test "cscli hub upgrade" { + #XXX: todo + : +} + +@test "cscli hub upgrade --force" { + #XXX: todo + : +} diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 694c92f8a93..5c2d85317d2 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -5,6 +5,7 @@ set -u setup_file() { load "../lib/setup_file.sh" + ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR CONFIG_DIR=$(config_get '.config_paths.config_dir') @@ -19,9 +20,8 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_uninstall_all - hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$HUB_DIR/.index.json" + hub_purge_all + hub_strip_index } teardown() { @@ -75,6 +75,8 @@ teardown() { rune -0 cscli collections list -o raw -a rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" + + # XXX: check alphabetical order in human, json, raw } @@ -159,6 +161,7 @@ teardown() { @test "cscli collections inspect [collection]..." { rune -1 cscli collections inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics ./instance-crowdsec start rune -1 cscli collections inspect blahblah/blahblah @@ -220,12 +223,17 @@ teardown() { @test "cscli collections remove [collection]..." { rune -1 cscli collections remove assert_stderr --partial "specify at least one collection to remove or '--all'" - rune -1 cscli collections remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in collections" - # XXX: we can however remove a real item if it's not installed, or already removed rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no removal required' + + rune -0 cscli collections install crowdsecurity/sshd --download-only + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'removing crowdsecurity/sshd: already uninstalled' + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_stderr --partial 'Removed source file [crowdsecurity/sshd]' # install, then remove, check files rune -0 cscli collections install crowdsecurity/sshd @@ -258,8 +266,8 @@ teardown() { assert_output "0" } -@test "cscli parsers remove [parser]... --force" { - # remove a parser that belongs to a collection +@test "cscli collections remove [collections]... --force" { + # remove a collections that belongs to a collection rune -0 cscli collections install crowdsecurity/linux rune -0 cscli collections remove crowdsecurity/sshd assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]" @@ -269,26 +277,24 @@ teardown() { @test "cscli collections upgrade [collection]..." { rune -1 cscli collections upgrade assert_stderr --partial "specify at least one collection to upgrade or '--all'" - - # XXX: should this return 1 instead of log.Error? - rune -0 cscli collections upgrade blahblah/blahblah + rune -1 cscli collections upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + rune -1 cscli collections upgrade crowdsecurity/exim + assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed" + rune -0 cscli collections install crowdsecurity/exim --download-only + rune -1 cscli collections upgrade crowdsecurity/exim + assert_stderr --partial "can't upgrade crowdsecurity/exim: downloaded but not installed" - # XXX: same message if the item exists but is not installed, this is confusing - rune -0 cscli collections upgrade crowdsecurity/sshd - assert_stderr --partial "can't find 'crowdsecurity/sshd' in collections" - - # hash of an empty file - sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - # add version 0.0 to the hub - new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/sshd":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + # add version 0.0 to all collections + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') echo "$new_hub" >"$HUB_DIR/.index.json" rune -0 cscli collections install crowdsecurity/sshd - # bring the file to v0.0 - truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml" + echo "v0.0" > "$CONFIG_DIR/collections/sshd.yaml" rune -0 cscli collections inspect crowdsecurity/sshd -o json rune -0 jq -e '.local_version=="0.0"' <(output) @@ -312,16 +318,20 @@ teardown() { # multiple items rune -0 cscli collections install crowdsecurity/smb - echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" - echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" rune -0 cscli collections list -o json - rune -0 jq -e '[.collections[].local_version]==["?","?"]' <(output) + rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb - rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) + rune -0 cscli collections list -o json + rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) # upgrade all - echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" - echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml" + echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli collections upgrade --all - rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) + rune -0 cscli collections list -o json + rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output) } diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index a703c65968d..18b940339fe 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -5,6 +5,7 @@ set -u setup_file() { load "../lib/setup_file.sh" + ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR CONFIG_DIR=$(config_get '.config_paths.config_dir') @@ -19,9 +20,8 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_uninstall_all - hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$HUB_DIR/.index.json" + hub_purge_all + hub_strip_index } teardown() { diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index b349cf4e2e0..a505b0de9fb 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -5,6 +5,7 @@ set -u setup_file() { load "../lib/setup_file.sh" + ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR CONFIG_DIR=$(config_get '.config_paths.config_dir') @@ -19,11 +20,8 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_uninstall_all - # XXX: remove all "content" fields from the index, to make sure - # XXX: we don't rely on it in any way - hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$HUB_DIR/.index.json" + hub_purge_all + hub_strip_index } teardown() { @@ -77,6 +75,8 @@ teardown() { rune -0 cscli parsers list -o raw -a rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" + + # XXX: check alphabetical order in human, json, raw } @test "cscli parsers list [parser]..." { @@ -164,6 +164,7 @@ teardown() { @test "cscli parsers inspect [parser]..." { rune -1 cscli parsers inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics ./instance-crowdsec start rune -1 cscli parsers inspect blahblah/blahblah @@ -224,18 +225,20 @@ teardown() { assert_output "0" } -@test "cscli parsers remove [parser]..." { +@test "foo cscli parsers remove [parser]..." { rune -1 cscli parsers remove assert_stderr --partial "specify at least one parser to remove or '--all'" - rune -1 cscli parsers remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" - # XXX: we can however remove a real item if it's not installed, or already removed rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no removal required' - # XXX: have the --force ignore uninstalled items - # XXX: maybe also with --purge + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial 'removing crowdsecurity/whitelists: already uninstalled' + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]' # install, then remove, check files rune -0 cscli parsers install crowdsecurity/whitelists @@ -279,26 +282,24 @@ teardown() { @test "cscli parsers upgrade [parser]..." { rune -1 cscli parsers upgrade assert_stderr --partial "specify at least one parser to upgrade or '--all'" - - # XXX: should this return 1 instead of log.Error? - rune -0 cscli parsers upgrade blahblah/blahblah + rune -1 cscli parsers upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + rune -1 cscli parsers upgrade crowdsecurity/pam-logs + assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed" + rune -0 cscli parsers install crowdsecurity/pam-logs --download-only + rune -1 cscli parsers upgrade crowdsecurity/pam-logs + assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: downloaded but not installed" - # XXX: same message if the item exists but is not installed, this is confusing - rune -0 cscli parsers upgrade crowdsecurity/whitelists - assert_stderr --partial "can't find 'crowdsecurity/whitelists' in parsers" - - # hash of an empty file - sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - # add version 0.0 to the hub - new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {parsers:{"crowdsecurity/whitelists":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + # add version 0.0 to all parsers + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') echo "$new_hub" >"$HUB_DIR/.index.json" rune -0 cscli parsers install crowdsecurity/whitelists - # bring the file to v0.0 - truncate -s 0 "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "v0.0" > "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" rune -0 cscli parsers inspect crowdsecurity/whitelists -o json rune -0 jq -e '.local_version=="0.0"' <(output) @@ -322,104 +323,20 @@ teardown() { # multiple items rune -0 cscli parsers install crowdsecurity/windows-auth - echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" rune -0 cscli parsers list -o json - rune -0 jq -e '[.parsers[].local_version]==["?","?"]' <(output) + rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth - rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) + rune -0 cscli parsers list -o json + rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) # upgrade all - echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" - echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli parsers upgrade --all - rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) + rune -0 cscli parsers list -o json + rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output) } - - - -#@test "must use --force to remove a collection that belongs to another, which becomes tainted" { -# # we expect no error since we may have multiple collections, some removed and some not -# rune -0 cscli collections remove crowdsecurity/sshd -# assert_stderr --partial "crowdsecurity/sshd belongs to other collections" -# assert_stderr --partial "[crowdsecurity/linux]" -# -# rune -0 cscli collections remove crowdsecurity/sshd --force -# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" -# rune -0 cscli collections inspect crowdsecurity/linux -o json -# rune -0 jq -r '.tainted' <(output) -# assert_output "true" -#} -# -#@test "can remove a collection" { -# rune -0 cscli collections remove crowdsecurity/linux -# assert_stderr --partial "Removed" -# assert_stderr --regexp ".*for the new configuration to be effective." -# rune -0 cscli collections inspect crowdsecurity/linux -o human --no-metrics -# assert_line 'installed: false' -#} -# -#@test "collections delete is an alias for collections remove" { -# rune -0 cscli collections delete crowdsecurity/linux -# assert_stderr --partial "Removed" -# assert_stderr --regexp ".*for the new configuration to be effective." -#} -# -#@test "removing a collection that does not exist is noop" { -# rune -0 cscli collections remove crowdsecurity/apache2 -# refute_stderr --partial "Removed" -# assert_stderr --regexp ".*for the new configuration to be effective." -#} -# -#@test "can remove a removed collection" { -# rune -0 cscli collections install crowdsecurity/mysql -# rune -0 cscli collections remove crowdsecurity/mysql -# assert_stderr --partial "Removed" -# rune -0 cscli collections remove crowdsecurity/mysql -# refute_stderr --partial "Removed" -#} -# -#@test "can remove all collections" { -# # we may have this too, from package installs -# rune cscli parsers delete crowdsecurity/whitelists -# rune -0 cscli collections remove --all -# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" -# assert_stderr --partial "Removed symlink [crowdsecurity/linux]" -# rune -0 cscli hub list -o json -# assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' -# rune -0 cscli collections remove --all -# assert_stderr --partial 'Disabled 0 items' -#} -# -#@test "a taint bubbles up to the top collection" { -# coll=crowdsecurity/nginx -# subcoll=crowdsecurity/base-http-scenarios -# scenario=crowdsecurity/http-crawl-non_statics -# -# # install a collection with dependencies -# rune -0 cscli collections install "$coll" -# -# # the collection, subcollection and scenario are installed and not tainted -# # we have to default to false because tainted is (as of 1.4.6) returned -# # only when true -# rune -0 cscli collections inspect "$coll" -o json -# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) -# rune -0 cscli collections inspect "$subcoll" -o json -# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) -# rune -0 cscli scenarios inspect "$scenario" -o json -# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) -# -# # we taint the scenario -# HUB_DIR=$(config_get '.config_paths.hub_dir') -# yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" -# -# # the collection, subcollection and scenario are now tainted -# rune -0 cscli scenarios inspect "$scenario" -o json -# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -# rune -0 cscli collections inspect "$subcoll" -o json -# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -# rune -0 cscli collections inspect "$coll" -o json -# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -#} -# -## TODO test download-only diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index da0f17b05e9..9dda68361d5 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -5,6 +5,7 @@ set -u setup_file() { load "../lib/setup_file.sh" + ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR CONFIG_DIR=$(config_get '.config_paths.config_dir') @@ -19,9 +20,8 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_uninstall_all - hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$HUB_DIR/.index.json" + hub_purge_all + hub_strip_index } teardown() { @@ -75,6 +75,8 @@ teardown() { rune -0 cscli postoverflows list -o raw -a rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" + + # XXX: check alphabetical order in human, json, raw } @@ -159,6 +161,7 @@ teardown() { @test "cscli postoverflows inspect [scenario]..." { rune -1 cscli postoverflows inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics ./instance-crowdsec start rune -1 cscli postoverflows inspect blahblah/blahblah @@ -222,12 +225,17 @@ teardown() { @test "cscli postoverflows remove [postoverflow]..." { rune -1 cscli postoverflows remove assert_stderr --partial "specify at least one postoverflow to remove or '--all'" - rune -1 cscli postoverflows remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" - # XXX: we can however remove a real item if it's not installed, or already removed rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no removal required' + + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial 'removing crowdsecurity/rdns: already uninstalled' + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_stderr --partial 'Removed source file [crowdsecurity/rdns]' # install, then remove, check files rune -0 cscli postoverflows install crowdsecurity/rdns @@ -260,8 +268,8 @@ teardown() { assert_output "0" } -@test "cscli postoverflows remove [parser]... --force" { - # remove a parser that belongs to a collection +@test "cscli postoverflows remove [postoverflow]... --force" { + # remove a postoverflow that belongs to a collection rune -0 cscli collections install crowdsecurity/auditd rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]" @@ -271,26 +279,24 @@ teardown() { @test "cscli postoverflows upgrade [postoverflow]..." { rune -1 cscli postoverflows upgrade assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" - - # XXX: should this return 1 instead of log.Error? - rune -0 cscli postoverflows upgrade blahblah/blahblah + rune -1 cscli postoverflows upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist + assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed" + rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only + rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist + assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: downloaded but not installed" - # XXX: same message if the item exists but is not installed, this is confusing - rune -0 cscli postoverflows upgrade crowdsecurity/rdns - assert_stderr --partial "can't find 'crowdsecurity/rdns' in postoverflows" - - # hash of an empty file - sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - # add version 0.0 to the hub - new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {postoverflows:{"crowdsecurity/rdns":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + # add version 0.0 to all postoverflows + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') echo "$new_hub" >"$HUB_DIR/.index.json" rune -0 cscli postoverflows install crowdsecurity/rdns - # bring the file to v0.0 - truncate -s 0 "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "v0.0" > "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json rune -0 jq -e '.local_version=="0.0"' <(output) @@ -314,16 +320,20 @@ teardown() { # multiple items rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist - echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" rune -0 cscli postoverflows list -o json - rune -0 jq -e '[.postoverflows[].local_version]==["?","?"]' <(output) + rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist - rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) + rune -0 cscli postoverflows list -o json + rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) # upgrade all - echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" - echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows list -o json + rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli postoverflows upgrade --all - rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) + rune -0 cscli postoverflows list -o json + rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output) } diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 04c344fe42d..29c73526e61 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -5,6 +5,7 @@ set -u setup_file() { load "../lib/setup_file.sh" + ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR CONFIG_DIR=$(config_get '.config_paths.config_dir') @@ -19,9 +20,8 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_uninstall_all - hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') - echo "$hub_min" >"$HUB_DIR/.index.json" + hub_purge_all + hub_strip_index } teardown() { @@ -75,6 +75,8 @@ teardown() { rune -0 cscli scenarios list -o raw -a rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" + + # XXX: check alphabetical order in human, json, raw } @test "cscli scenarios list [scenario]..." { @@ -161,6 +163,7 @@ teardown() { @test "cscli scenarios inspect [scenario]..." { rune -1 cscli scenarios inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' + # required for metrics ./instance-crowdsec start rune -1 cscli scenarios inspect blahblah/blahblah @@ -222,12 +225,17 @@ teardown() { @test "cscli scenarios remove [scenario]..." { rune -1 cscli scenarios remove assert_stderr --partial "specify at least one scenario to remove or '--all'" - rune -1 cscli scenarios remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" - # XXX: we can however remove a real item if it's not installed, or already removed rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no removal required' + + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial 'removing crowdsecurity/ssh-bf: already uninstalled' + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]' # install, then remove, check files rune -0 cscli scenarios install crowdsecurity/ssh-bf @@ -271,26 +279,24 @@ teardown() { @test "cscli scenarios upgrade [scenario]..." { rune -1 cscli scenarios upgrade assert_stderr --partial "specify at least one scenario to upgrade or '--all'" - - # XXX: should this return 1 instead of log.Error? - rune -0 cscli scenarios upgrade blahblah/blahblah + rune -1 cscli scenarios upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf + assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed" + rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only + rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf + assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: downloaded but not installed" - # XXX: same message if the item exists but is not installed, this is confusing - rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf - assert_stderr --partial "can't find 'crowdsecurity/ssh-bf' in scenarios" - - # hash of an empty file - sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + # hash of the string "v0.0" + sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" - # add version 0.0 to the hub - new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {scenarios:{"crowdsecurity/ssh-bf":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + # add version 0.0 to all scenarios + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') echo "$new_hub" >"$HUB_DIR/.index.json" rune -0 cscli scenarios install crowdsecurity/ssh-bf - # bring the file to v0.0 - truncate -s 0 "$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "v0.0" > "$CONFIG_DIR/scenarios/ssh-bf.yaml" rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json rune -0 jq -e '.local_version=="0.0"' <(output) @@ -314,16 +320,20 @@ teardown() { # multiple items rune -0 cscli scenarios install crowdsecurity/telnet-bf - echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" rune -0 cscli scenarios list -o json - rune -0 jq -e '[.scenarios[].local_version]==["?","?"]' <(output) + rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf - rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) + rune -0 cscli scenarios list -o json + rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) # upgrade all - echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" - echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios list -o json + rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output) rune -0 cscli scenarios upgrade --all - rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) + rune -0 cscli scenarios list -o json + rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output) } diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index bcd216091f1..7d298418adf 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -238,11 +238,29 @@ assert_stderr_line() { } export -f assert_stderr_line -hub_uninstall_all() { +# remove all installed items and data +hub_purge_all() { + local CONFIG_DIR CONFIG_DIR=$(dirname "$CONFIG_YAML") rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/* + rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/* + local DATA_DIR + DATA_DIR=$(config_get .config_paths.data_dir) + # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete), + # but don't play with fire if there is a misconfiguration + rm -rfv "$DATA_DIR"/GeoLite* } -export -f hub_uninstall_all +export -f hub_purge_all + +# remove unused data from the index, to make sure we don't rely on it in any way +hub_strip_index() { + local INDEX + INDEX=$(config_get .config_paths.index_path) + local hub_min + hub_min=$(jq <"$INDEX" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$INDEX" +} +export -f hub_strip_index # remove color and style sequences from stdin plaintext() { From 84ffde18449381907302a84ec3a9fa075030eef9 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 31 Oct 2023 11:53:13 +0100 Subject: [PATCH 149/263] add body_type in custom rule --- pkg/waf/waap_rule/modsecurity.go | 15 +++++++++++++++ pkg/waf/waap_rule/waap_rule.go | 1 + 2 files changed, 16 insertions(+) diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go index b7ff3242807..8b3ab9f9ffe 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/waf/waap_rule/modsecurity.go @@ -43,6 +43,13 @@ var matchMap map[string]string = map[string]string{ "le": "@le", } +var bodyTypeMatch map[string]string = map[string]string{ + "json": "JSON", + "xml": "XML", + "multipart": "MULTIPART", + "urlencoded": "URLENCODED", +} + func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, []uint32, error) { rules, err := m.buildRules(rule, waapRuleName, false, 0) @@ -146,6 +153,14 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and } } + if rule.BodyType != "" { + if mappedBodyType, ok := bodyTypeMatch[rule.BodyType]; ok { + r.WriteString(fmt.Sprintf(",ctl:requestBodyProcessor=%s", mappedBodyType)) + } else { + return nil, fmt.Errorf("unknown body type '%s'", rule.BodyType) + } + } + if and { r.WriteString(",chain") } diff --git a/pkg/waf/waap_rule/waap_rule.go b/pkg/waf/waap_rule/waap_rule.go index 279ff0a0bc9..00e6cefd113 100644 --- a/pkg/waf/waap_rule/waap_rule.go +++ b/pkg/waf/waap_rule/waap_rule.go @@ -40,6 +40,7 @@ type CustomRule struct { Transform []string `yaml:"transform"` //t:lowercase, t:uppercase, etc And []CustomRule `yaml:"and,omitempty"` Or []CustomRule `yaml:"or,omitempty"` + BodyType string `yaml:"body_type,omitempty"` } func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, []uint32, error) { From 590a19b768866bd2ed46133ac7a3117781ed920f Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 31 Oct 2023 12:47:39 +0100 Subject: [PATCH 150/263] Refact pkg/cwhub: constructor, cscli output * Single constructor: NewHub() to replace InitHub(), InitHubUpdate() * sort cscli hub list output * log.Fatal -> fmt.Errorf --- cmd/crowdsec-cli/hub.go | 10 +-- cmd/crowdsec-cli/items.go | 26 +++--- cmd/crowdsec-cli/require/require.go | 2 +- cmd/crowdsec/crowdsec.go | 2 +- pkg/cwhub/cwhub_test.go | 13 +-- pkg/cwhub/helpers_test.go | 14 ++-- pkg/cwhub/hub.go | 126 +++------------------------- pkg/cwhub/hub_test.go | 42 ++++------ pkg/cwhub/remote.go | 95 +++++++++++++++++++++ pkg/hubtest/hubtest_item.go | 2 +- pkg/leakybucket/buckets_test.go | 4 +- test/bats/20_hub.bats | 6 +- 12 files changed, 162 insertions(+), 180 deletions(-) create mode 100644 pkg/cwhub/remote.go diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 79cf4190e2f..447462ad7dd 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -64,9 +64,7 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(line) } - err = ListItems(hub, color.Output, []string{ - cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, - }, nil, true, false, all) + err = ListItems(hub, color.Output, cwhub.ItemTypes, nil, true, false, all) if err != nil { return err } @@ -94,16 +92,16 @@ func runHubUpdate(cmd *cobra.Command, args []string) error { remote := require.RemoteHub(csConfig) // don't use require.Hub because if there is no index file, it would fail - hub, err := cwhub.InitHubUpdate(local, remote) + hub, err := cwhub.NewHub(local, remote, true) if err != nil { // XXX: this should be done when downloading items, too // but what is the fallback to master actually solving? if !errors.Is(err, cwhub.ErrIndexNotFound) { - return fmt.Errorf("failed to get Hub index : %w", err) + return fmt.Errorf("failed to get Hub index: %w", err) } log.Warnf("Could not find index file for branch '%s', using 'master'", remote.Branch) remote.Branch = "master" - if hub, err = cwhub.InitHubUpdate(local, remote); err != nil { + if hub, err = cwhub.NewHub(local, remote, true); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 46c5a0c9e1c..444f9967760 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -9,7 +9,6 @@ import ( "sort" "strings" - log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" "slices" @@ -50,20 +49,22 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b } func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { - var err error - items := make(map[string][]string) for _, itemType := range itemTypes { - if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil { + selected, err := selectItems(hub, itemType, args, !all) + if err != nil { return err } + sort.Strings(selected) + items[itemType] = selected } - if csConfig.Cscli.Output == "human" { + switch csConfig.Cscli.Output { + case "human": for _, itemType := range itemTypes { listHubItemTable(hub, out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) } - } else if csConfig.Cscli.Output == "json" { + case "json": type itemHubStatus struct { Name string `json:"name"` LocalVersion string `json:"local_version"` @@ -89,15 +90,13 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, UTF8Status: fmt.Sprintf("%v %s", emo, status), } } - h := hubStatus[itemType] - sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name }) } x, err := json.MarshalIndent(hubStatus, "", " ") if err != nil { - log.Fatalf("failed to unmarshal") + return fmt.Errorf("failed to unmarshal: %w", err) } out.Write(x) - } else if csConfig.Cscli.Output == "raw" { + case "raw": csvwriter := csv.NewWriter(out) if showHeader { header := []string{"name", "status", "version", "description"} @@ -106,7 +105,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, } err := csvwriter.Write(header) if err != nil { - log.Fatalf("failed to write header: %s", err) + return fmt.Errorf("failed to write header: %s", err) } } for _, itemType := range itemTypes { @@ -127,12 +126,15 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, } err := csvwriter.Write(row) if err != nil { - log.Fatalf("failed to write raw output : %s", err) + return fmt.Errorf("failed to write raw output: %s", err) } } } csvwriter.Flush() + default: + return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output) } + return nil } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index d87decc5679..855b27a7cee 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -86,7 +86,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) { return nil, fmt.Errorf("you must configure cli before interacting with hub") } - hub, err := cwhub.InitHub(local, remote) + hub, err := cwhub.NewHub(local, remote, false) if err != nil { return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 1b688f2d53e..3b3a69cd33f 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,7 +23,7 @@ import ( func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error - hub, err := cwhub.InitHub(cConfig.Hub, nil) + hub, err := cwhub.NewHub(cConfig.Hub, nil, false) if err != nil { return nil, fmt.Errorf("while loading hub index: %w", err) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 1b56bfebd8f..3e1bb57d151 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -63,18 +63,13 @@ func testHub(t *testing.T, update bool) *Hub { var hub *Hub remote := &RemoteHubCfg{ - Branch: "master", + Branch: "master", URLTemplate: mockURLTemplate, - IndexPath: ".index.json", + IndexPath: ".index.json", } - if update { - hub, err = InitHubUpdate(local, remote) - require.NoError(t, err) - } else { - hub, err = InitHub(local, remote) - require.NoError(t, err) - } + hub, err = NewHub(local, remote, update) + require.NoError(t, err) return hub } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index f6ea249545a..a859c116d6f 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -36,10 +36,10 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { remote := &RemoteHubCfg{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", + IndexPath: ".index.json", } - hub, err := InitHubUpdate(hub.local, remote) + hub, err := NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) hub = getHubOrFail(t, hub.local, remote) @@ -84,7 +84,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { remote := &RemoteHubCfg{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", + IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) @@ -95,7 +95,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - hub, err = InitHubUpdate(hub.local, remote) + hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) @@ -108,7 +108,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { - hub, err := InitHub(local, remote) + hub, err := NewHub(local, remote, false) require.NoError(t, err, "failed to load hub index") return hub @@ -141,7 +141,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te remote := &RemoteHubCfg{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", + IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) @@ -158,7 +158,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - hub, err = InitHubUpdate(hub.local, remote) + hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index d695f0b8b7b..7995389ccab 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -1,12 +1,9 @@ package cwhub import ( - "bytes" "encoding/json" "errors" "fmt" - "io" - "net/http" "os" "strings" @@ -15,15 +12,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -// const HubIndexFile = ".index.json" - -// RemoteHubCfg contains where to find the remote hub, which branch etc. -type RemoteHubCfg struct { - Branch string - URLTemplate string - IndexPath string -} - type Hub struct { Items HubItems local *csconfig.LocalHubCfg @@ -48,12 +36,20 @@ func GetHub() (*Hub, error) { return theHub, nil } -// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use -func InitHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { +// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state +// It also downloads the index if downloadIndex is true +func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex bool) (*Hub, error) { if local == nil { return nil, fmt.Errorf("no hub configuration found") } + if downloadIndex { + err := remote.DownloadIndex(local.HubIndexFile) + if err != nil { + return nil, err + } + } + log.Debugf("loading hub idx %s", local.HubIndexFile) bidx, err := os.ReadFile(local.HubIndexFile) @@ -64,7 +60,7 @@ func InitHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("unable to load existing index: %w", err) + return nil, fmt.Errorf("failed to load index: %w", err) } // XXX: why the error check if we bail out anyway? @@ -79,110 +75,12 @@ func InitHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { _, err = theHub.LocalSync() if err != nil { - return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) - } - - return theHub, nil -} - -// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk -// It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) { - if local == nil { - return nil, fmt.Errorf("no configuration found for hub") - } - - bidx, err := remote.DownloadIndex(local.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("failed to download index: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("failed to read index: %w", err) - } - } - - theHub = &Hub{ - Items: ret, - local: local, - remote: remote, - } - - if _, err := theHub.LocalSync(); err != nil { - return nil, fmt.Errorf("failed to sync: %w", err) + return nil, fmt.Errorf("failed to sync hub index: %w", err) } return theHub, nil } -func (r RemoteHubCfg) urlTo(remotePath string) (string, error) { - if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { - return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) - } - - return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil -} - -// DownloadIndex downloads the latest version of the index and returns the content -func (r RemoteHubCfg) DownloadIndex(localPath string) ([]byte, error) { - url, err := r.urlTo(r.IndexPath) - if err != nil { - return nil, fmt.Errorf("failed to build hub index request: %w", err) - } - - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("failed to build request for hub index: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed http request for hub index: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return nil, ErrIndexNotFound - } - - return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) - } - - oldContent, err := os.ReadFile(localPath) - if err != nil { - if !os.IsNotExist(err) { - log.Warningf("failed to read hub index: %s", err) - } - } else if bytes.Equal(body, oldContent) { - log.Info("hub index is up to date") - return body, nil - } - - file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) - - if err != nil { - return nil, fmt.Errorf("while opening hub index file: %w", err) - } - defer file.Close() - - wsize, err := file.Write(body) - if err != nil { - return nil, fmt.Errorf("while writing hub index file: %w", err) - } - - log.Infof("Wrote index to %s, %d bytes", localPath, wsize) - - return body, nil -} - // ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections func ParseIndex(buff []byte) (HubItems, error) { var ( diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index b4b0cd7d4f5..acaa0ec31c8 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,13 +13,13 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - remote := &RemoteHubCfg { + remote := &RemoteHubCfg{ URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", + Branch: "master", + IndexPath: ".index.json", } - _, err := InitHubUpdate(hub.local, remote) + _, err := NewHub(hub.local, remote, true) require.NoError(t, err) _, err = GetHub() @@ -39,43 +39,37 @@ func TestDownloadIndex(t *testing.T) { hub := envSetup(t) - hub.remote = &RemoteHubCfg { + hub.remote = &RemoteHubCfg{ URLTemplate: "x", - Branch: "", - IndexPath: "", + Branch: "", + IndexPath: "", } - ret, err := hub.remote.DownloadIndex(tmpIndex.Name()) + err = hub.remote.DownloadIndex(tmpIndex.Name()) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") - fmt.Printf("->%+v", ret) - // bad domain fmt.Println("Test 'bad domain'") - hub.remote = &RemoteHubCfg { + hub.remote = &RemoteHubCfg{ URLTemplate: "https://baddomain/%s/%s", - Branch: "master", - IndexPath: ".index.json", + Branch: "master", + IndexPath: ".index.json", } - ret, err = hub.remote.DownloadIndex(tmpIndex.Name()) -// XXX: this is not failing -// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") - - fmt.Printf("->%+v", ret) + err = hub.remote.DownloadIndex(tmpIndex.Name()) + // XXX: this is not failing + // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") // bad target path fmt.Println("Test 'bad target path'") - hub.remote = &RemoteHubCfg { + hub.remote = &RemoteHubCfg{ URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", + Branch: "master", + IndexPath: ".index.json", } - ret, err = hub.remote.DownloadIndex("/does/not/exist/index.json") + err = hub.remote.DownloadIndex("/does/not/exist/index.json") cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") - - fmt.Printf("->%+v", ret) } diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go new file mode 100644 index 00000000000..4ff2a6d5ea6 --- /dev/null +++ b/pkg/cwhub/remote.go @@ -0,0 +1,95 @@ +package cwhub + +import ( + "bytes" + "fmt" + "io" + "net/http" + "os" + + log "github.com/sirupsen/logrus" +) + +// RemoteHubCfg contains where to find the remote hub, which branch etc. +type RemoteHubCfg struct { + Branch string + URLTemplate string + IndexPath string +} + +// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. All attempts to download index or items will return this error. +var ErrNilRemoteHub = fmt.Errorf("remote hub configuration is not provided. Please report this issue to the developers") + +func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { + if r == nil { + return "", ErrNilRemoteHub + } + + if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { + return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) + } + + return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil +} + +// DownloadIndex downloads the latest version of the index +func (r *RemoteHubCfg) DownloadIndex(localPath string) error { + if r == nil { + return ErrNilRemoteHub + } + + url, err := r.urlTo(r.IndexPath) + if err != nil { + return fmt.Errorf("failed to build hub index request: %w", err) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to build request for hub index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed http request for hub index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return ErrIndexNotFound + } + + return fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read request answer for hub index: %w", err) + } + + oldContent, err := os.ReadFile(localPath) + if err != nil { + if !os.IsNotExist(err) { + log.Warningf("failed to read hub index: %s", err) + } + } else if bytes.Equal(body, oldContent) { + log.Info("hub index is up to date") + return nil + } + + file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + + if err != nil { + return fmt.Errorf("while opening hub index file: %w", err) + } + defer file.Close() + + wsize, err := file.Write(body) + if err != nil { + return fmt.Errorf("while writing hub index file: %w", err) + } + + log.Infof("Wrote index to %s, %d bytes", localPath, wsize) + + return nil +} diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index e9ffebe0a41..44018a14acb 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -391,7 +391,7 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - hub, err := cwhub.InitHub(t.RuntimeHubConfig, nil) + hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false) if err != nil { log.Fatal(err) } diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index 7e3f3e98817..b47b0717ea0 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -43,9 +43,9 @@ func TestBucket(t *testing.T) { HubIndexFile: filepath.Join(testdata, "hub", "index.json"), } - _, err := cwhub.InitHub(hubCfg, nil) + _, err := cwhub.NewHub(hubCfg, nil, false) if err != nil { - t.Fatalf("failed to init hub : %s", err) + t.Fatalf("failed to init hub: %s", err) } err = exprhelpers.Init(nil) diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index c0f10572bb4..1dc4fd87212 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -33,7 +33,7 @@ teardown() { @test "cscli hub list" { # no items rune -0 cscli hub list - assert_output --regexp ".*COLLECTIONS.*PARSERS.*SCENARIOS.*POSTOVERFLOWS.*" + assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" rune -0 cscli hub list -o json assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw @@ -43,7 +43,7 @@ teardown() { rune -0 cscli parsers install crowdsecurity/whitelists rune -0 cscli scenarios install crowdsecurity/telnet-bf rune -0 cscli hub list - assert_output --regexp ".*COLLECTIONS.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*" + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*" rune -0 cscli hub list -o json rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output) rune -0 cscli hub list -o raw @@ -53,7 +53,7 @@ teardown() { # all items rune -0 cscli hub list -a - assert_output --regexp ".*COLLECTIONS.*crowdsecurity/linux.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*" + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/linux.*" rune -0 cscli hub list -a -o json rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) rune -0 cscli hub list -a -o raw From fcd6c468c4089826ae2bb3a46474bd7d5eb7b6fc Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 31 Oct 2023 13:12:14 +0100 Subject: [PATCH 151/263] fix lint --- pkg/cwhub/hub_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index acaa0ec31c8..779bc25425a 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -58,6 +58,7 @@ func TestDownloadIndex(t *testing.T) { } err = hub.remote.DownloadIndex(tmpIndex.Name()) + require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") From 450c263826a210b5bdb387a2bfe4f3e74b702a8c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:32:29 +0100 Subject: [PATCH 152/263] Refact cwhub: minor cleanups and comments (#2574) * check response status before body; close file * err check one-liners, lint, comments * simplify function logic, reduce code * comments, xxx, whitespace --- cmd/crowdsec-cli/item_suggest.go | 49 +++++++++++++------------------- cmd/crowdsec-cli/itemcommands.go | 15 ++++++---- cmd/crowdsec-cli/items.go | 27 +++++++----------- pkg/cwhub/cwhub.go | 6 ---- pkg/cwhub/dataset.go | 25 +++++++--------- pkg/cwhub/enable.go | 19 ++++++------- pkg/cwhub/enable_test.go | 4 +-- pkg/cwhub/errors.go | 12 ++++++++ pkg/cwhub/helpers.go | 46 ++++++++++++++---------------- pkg/cwhub/hub.go | 6 ++-- pkg/cwhub/items.go | 2 ++ pkg/cwhub/items_test.go | 4 +-- pkg/cwhub/remote.go | 5 +--- pkg/cwhub/sync.go | 21 +++++++------- 14 files changed, 110 insertions(+), 131 deletions(-) create mode 100644 pkg/cwhub/errors.go diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index 27a078eb6ca..c9dc09eba7d 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/agext/levenshtein" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "slices" @@ -15,38 +14,26 @@ import ( const MaxDistance = 7 -func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { - errMsg := "" - if score < MaxDistance { - errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) - } else { - errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) - } - if ignoreErr { - log.Error(errMsg) - } else { - log.Fatalf(errMsg) +// SuggestNearestMessage returns a message with the most similar item name, if one is found +func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string { + score := 100 + nearest := "" + + for _, item := range hub.GetItemMap(itemType) { + d := levenshtein.Distance(itemName, item.Name, nil) + if d < score { + score = d + nearest = item.Name + } } -} -func GetDistance(hub *cwhub.Hub, itemType string, itemName string) (*cwhub.Item, int) { - allItems := make([]string, 0) - nearestScore := 100 - nearestItem := &cwhub.Item{} + msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType) - hubItems := hub.GetItemMap(itemType) - for _, item := range hubItems { - allItems = append(allItems, item.Name) + if score < MaxDistance { + msg += fmt.Sprintf(", did you mean '%s'?", nearest) } - for _, s := range allItems { - d := levenshtein.Distance(itemName, s, nil) - if d < nearestScore { - nearestScore = d - nearestItem = hub.GetItem(itemType, s) - } - } - return nearestItem, nearestScore + return msg } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -56,13 +43,15 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string, } comp := make([]string, 0) - hubItems := hub.GetItemMap(itemType) - for _, item := range hubItems { + + for _, item := range hub.GetItemMap(itemType) { if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { comp = append(comp, item.Name) } } + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + return comp, cobra.ShellCompDirectiveNoFileComp } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index a4e902afdf8..5a31dda2e36 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -203,11 +203,13 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) } for _, name := range args { - t := hub.GetItem(it.name, name) - if t == nil { - nearestItem, score := GetDistance(hub, it.name, name) - Suggest(it.name, name, nearestItem.Name, score, ignoreError) + if hub.GetItem(it.name, name) == nil { + msg := SuggestNearestMessage(hub, it.name, name) + if !ignoreError { + return fmt.Errorf(msg) + } + log.Errorf(msg) continue } @@ -218,6 +220,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) log.Errorf("Error while installing '%s': %s", name, err) } } + // XXX: only reload if we installed something log.Infof(ReloadMessage()) return nil @@ -281,6 +284,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e } removed := 0 + for _, item := range items { didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force) if err != nil { @@ -290,6 +294,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e removed++ } } + log.Infof("Removed %d %s", removed, it.name) if removed > 0 { log.Infof(ReloadMessage()) @@ -478,7 +483,7 @@ func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) if item == nil { return fmt.Errorf("can't find '%s' in %s", name, it.name) } - if err = InspectItem(hub, item, noMetrics); err != nil { + if err = InspectItem(hub, item, !noMetrics); err != nil { return err } } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 444f9967760..9e13eb17937 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -19,6 +19,7 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b itemNames := hub.GetItemNames(itemType) notExist := []string{} + if len(args) > 0 { for _, arg := range args { if !slices.Contains(itemNames, arg) { @@ -98,6 +99,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, out.Write(x) case "raw": csvwriter := csv.NewWriter(out) + if showHeader { header := []string{"name", "status", "version", "description"} if showType { @@ -124,8 +126,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, if showType { row = append(row, itemType) } - err := csvwriter.Write(row) - if err != nil { + if err := csvwriter.Write(row); err != nil { return fmt.Errorf("failed to write raw output: %s", err) } } @@ -138,34 +139,26 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, return nil } -func InspectItem(hub *cwhub.Hub, item *cwhub.Item, noMetrics bool) error { - var ( - b []byte - err error - ) - +func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error { switch csConfig.Cscli.Output { case "human", "raw": enc := yaml.NewEncoder(os.Stdout) enc.SetIndent(2) - err = enc.Encode(item) - if err != nil { + if err := enc.Encode(item); err != nil { return fmt.Errorf("unable to encode item: %s", err) } case "json": - b, err = json.MarshalIndent(*item, "", " ") + b, err := json.MarshalIndent(*item, "", " ") if err != nil { return fmt.Errorf("unable to marshal item: %s", err) } - fmt.Printf("%s", string(b)) + fmt.Print(string(b)) } - if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return nil + if csConfig.Cscli.Output == "human" && showMetrics { + fmt.Printf("\nCurrent metrics: \n") + ShowMetrics(hub, item) } - fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hub, item) - return nil } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 24c2d859eca..8a68c97a5f9 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -3,9 +3,3 @@ // This includes retrieving the index, the items to install (parsers, scenarios, data files...) // and managing the dependencies and taints. package cwhub - -import ( - "errors" -) - -var ErrMissingReference = errors.New("Reference(s) missing in collection") diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index e031fde3a93..802c5775c14 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -32,27 +32,27 @@ func downloadFile(url string, destPath string) error { } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP status '%s' from %s", resp.Status, url) + } + body, err := io.ReadAll(resp.Body) if err != nil { return err } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) - } - file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return err } + defer file.Close() _, err = file.Write(body) if err != nil { return err } - err = file.Sync() - if err != nil { + if err = file.Sync(); err != nil { return err } @@ -64,8 +64,7 @@ func GetData(data []*types.DataSource, dataDir string) error { destPath := filepath.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) - err := downloadFile(dataS.SourceURL, destPath) - if err != nil { + if err := downloadFile(dataS.SourceURL, destPath); err != nil { return err } } @@ -75,15 +74,12 @@ func GetData(data []*types.DataSource, dataDir string) error { // downloadData downloads the data files for an item func downloadData(dataFolder string, force bool, reader io.Reader) error { - var err error - dec := yaml.NewDecoder(reader) for { data := &DataSet{} - err = dec.Decode(data) - if err != nil { + if err := dec.Decode(data); err != nil { if errors.Is(err, io.EOF) { break } @@ -94,14 +90,13 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { download := false for _, dataS := range data.Data { - if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { download = true } } if download || force { - err = GetData(data.Data, dataFolder) - if err != nil { + if err := GetData(data.Data, dataFolder); err != nil { return fmt.Errorf("while getting data: %w", err) } } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 0e60212adb8..38c547e0139 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -14,8 +14,6 @@ import ( // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively func (h *Hub) EnableItem(target *Item) error { - var err error - parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") // create directories if needed @@ -35,7 +33,7 @@ func (h *Hub) EnableItem(target *Item) error { } } - if _, err = os.Stat(parentDir); os.IsNotExist(err) { + if _, err := os.Stat(parentDir); os.IsNotExist(err) { log.Infof("%s doesn't exist, create", parentDir) if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { @@ -51,15 +49,14 @@ func (h *Hub) EnableItem(target *Item) error { return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) } - err = h.EnableItem(&val) - if err != nil { + if err := h.EnableItem(&val); err != nil { return fmt.Errorf("while installing %s: %w", sub.Name, err) } } } // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { + if _, err := os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { log.Infof("%s already exists.", parentDir+"/"+target.FileName) return nil } @@ -79,7 +76,7 @@ func (h *Hub) EnableItem(target *Item) error { return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) } - log.Infof("Enabled %s : %s", target.Type, target.Name) + log.Infof("Enabled %s: %s", target.Type, target.Name) target.Installed = true h.Items[target.Type][target.Name] = *target @@ -103,6 +100,7 @@ func (h *Hub) purgeItem(target Item) (Item, error) { // DisableItem to disable an item managed by the hub, removes the symlink if purge is true func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { + // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not var err error // already disabled, noop unless purge @@ -145,8 +143,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { } if toRemove { - err = h.DisableItem(&val, purge, force) - if err != nil { + if err = h.DisableItem(&val, purge, force); err != nil { return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } else { @@ -164,7 +161,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { if os.IsNotExist(err) { // we only accept to "delete" non existing items if it's a forced purge if !purge && !force { - return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml) + return fmt.Errorf("can't delete %s: %s doesn't exist", target.Name, syml) } } else { // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... @@ -193,7 +190,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return fmt.Errorf("while removing symlink: %w", err) } - log.Infof("Removed symlink [%s] : %s", target.Name, syml) + log.Infof("Removed symlink [%s]: %s", target.Name, syml) } target.Installed = false diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 864b3fa8587..8b7cac88db6 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -72,7 +72,7 @@ func testDisable(hub *Hub, t *testing.T, item Item) { // Local sync and check status warns, err := hub.LocalSync() require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) + require.Empty(t, warns, "unexpected warnings: %+v", warns) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) @@ -85,7 +85,7 @@ func testDisable(hub *Hub, t *testing.T, item Item) { // Local sync and check status warns, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings : %+v", warns) + require.Empty(t, warns, "unexpected warnings: %+v", warns) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go new file mode 100644 index 00000000000..7f39ace8ce0 --- /dev/null +++ b/pkg/cwhub/errors.go @@ -0,0 +1,12 @@ +package cwhub + +import ( + "errors" +) + +var ( + // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. + // All attempts to download index or items will return this error. + ErrMissingReference = errors.New("Reference(s) missing in collection") + ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") +) diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index aae2f23dd1e..0357259edc9 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -34,22 +34,24 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } } - err := h.DownloadLatest(item, force, true) - if err != nil { + // XXX: confusing semantic between force and updateOnly? + if err := h.DownloadLatest(item, force, true); err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = h.AddItem(*item); err != nil { + if err := h.AddItem(*item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } if downloadOnly { + // XXX: should get the path from DownloadLatest log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath)) return nil } - err = h.EnableItem(item) - if err != nil { + // XXX: should we stop here if the item is already installed? + + if err := h.EnableItem(item); err != nil { return fmt.Errorf("while enabling %s: %w", item.Name, err) } @@ -153,13 +155,12 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error // DownloadLatest will download the latest version of Item to the tdir directory func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { - var err error - + // XXX: should return the path of the downloaded file (taken from DownloadItem) log.Debugf("Downloading %s %s", target.Type, target.Name) if target.Type != COLLECTIONS { if !target.Installed && updateOnly && target.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) + log.Debugf("skipping upgrade of %s: not installed", target.Name) return nil } @@ -174,32 +175,31 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro } if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s : not installed", target.Name) + log.Debugf("skipping upgrade of %s: not installed", target.Name) continue } - log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) - //recurse as it's a collection + log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) + + // recurse as it's a collection if sub.Type == COLLECTIONS { log.Tracef("collection, recurse") - err = h.DownloadLatest(&val, overwrite, updateOnly) - if err != nil { + if err := h.DownloadLatest(&val, overwrite, updateOnly); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } } downloaded := val.Downloaded - err = h.DownloadItem(&val, overwrite) - if err != nil { + if err := h.DownloadItem(&val, overwrite); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { - if err = h.EnableItem(&val); err != nil { + if err := h.EnableItem(&val); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } @@ -207,8 +207,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro h.Items[sub.Type][sub.Name] = val } - err = h.DownloadItem(target, overwrite) - if err != nil { + if err := h.DownloadItem(target, overwrite); err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -226,13 +225,13 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { if target.Tainted { - log.Debugf("%s : tainted, not updated", target.Name) + log.Debugf("%s: tainted, not updated", target.Name) return nil } if target.UpToDate { // We still have to check if data files are present - log.Debugf("%s : up-to-date, not updated", target.Name) + log.Debugf("%s: up-to-date, not updated", target.Name) } } @@ -245,13 +244,12 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { if err != nil { return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("while reading %s: %w", req.URL.String(), err) @@ -296,10 +294,10 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { // check actual file if _, err = os.Stat(finalPath); !os.IsNotExist(err) { - log.Warningf("%s : overwrite", target.Name) + log.Warningf("%s: overwrite", target.Name) log.Debugf("target: %s/%s", tdir, target.RemotePath) } else { - log.Infof("%s : OK", target.Name) + log.Infof("%s: OK", target.Name) } f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 7995389ccab..1aaa0ca0e32 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -44,8 +44,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo } if downloadIndex { - err := remote.DownloadIndex(local.HubIndexFile) - if err != nil { + if err := remote.DownloadIndex(local.HubIndexFile); err != nil { return nil, err } } @@ -73,8 +72,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo remote: remote, } - _, err = theHub.LocalSync() - if err != nil { + if _, err = theHub.LocalSync(); err != nil { return nil, fmt.Errorf("failed to sync hub index: %w", err) } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index ff890d84ef3..f5981216d4a 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -76,6 +76,7 @@ func (i *Item) IsLocal() bool { // It must not use a pointer receiver func (i Item) MarshalJSON() ([]byte, error) { type Alias Item + return json.Marshal(&struct { Alias Local bool `json:"local"` // XXX: omitempty? @@ -90,6 +91,7 @@ func (i Item) MarshalJSON() ([]byte, error) { // It must not use a pointer receiver func (i Item) MarshalYAML() (interface{}, error) { type Alias Item + return &struct { Alias `yaml:",inline"` Local bool `yaml:"local"` diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index 9971771e97b..430d6a5746d 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -15,7 +15,7 @@ func TestItemStatus(t *testing.T) { x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) - // Get item : good and bad + // Get item: good and bad for k := range x { item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) @@ -52,7 +52,7 @@ func TestGetters(t *testing.T) { x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) - // Get item : good and bad + // Get item: good and bad for k := range x { empty := hub.GetItem(COLLECTIONS, k+"nope") require.Nil(t, empty) diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 4ff2a6d5ea6..8117cd1a64d 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -17,9 +17,6 @@ type RemoteHubCfg struct { IndexPath string } -// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. All attempts to download index or items will return this error. -var ErrNilRemoteHub = fmt.Errorf("remote hub configuration is not provided. Please report this issue to the developers") - func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { if r == nil { return "", ErrNilRemoteHub @@ -59,7 +56,7 @@ func (r *RemoteHubCfg) DownloadIndex(localPath string) error { return ErrIndexNotFound } - return fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String()) + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) } body, err := io.ReadAll(resp.Body) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 7dd04ff2c0f..c5d87613bf3 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -81,7 +81,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { //.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { - return itemFileInfo{}, false, fmt.Errorf("path is too short : %s (%d)", path, len(subs)) + return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ret.fname = subs[len(subs)-1] @@ -115,7 +115,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { ret.ftype = COLLECTIONS ret.stage = "" } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { - // its a PARSER / POSTOVERFLOW with a stage + // it's a PARSER / POSTOVERFLOW with a stage return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) } @@ -205,7 +205,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { match := false for name, item := range h.Items[info.ftype] { - log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") + log.Tracef("check [%s] vs [%s]: %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") if info.fname != item.FileName { log.Tracef("%s != %s (filename)", info.fname, item.FileName) @@ -241,10 +241,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { sha, err := getSHA256(path) if err != nil { - log.Fatalf("Failed to get sha of %s : %v", path, err) + log.Fatalf("Failed to get sha of %s: %v", path, err) } // let's reverse sort the versions to deal with hash collisions (#154) + // XXX: we sure, lexical sorting? versions := make([]string, 0, len(item.Versions)) for k := range item.Versions { versions = append(versions, k) @@ -253,8 +254,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { sort.Sort(sort.Reverse(sort.StringSlice(versions))) for _, version := range versions { - val := item.Versions[version] - if sha != val.Digest { + if item.Versions[version].Digest != sha { // log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) continue } @@ -287,6 +287,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { log.Tracef("got tainted match for %s: %s", item.Name, path) h.skippedTainted++ + // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { item.LocalPath = path @@ -400,8 +401,7 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { continue } - err = filepath.WalkDir(cpath, h.itemVisit) - if err != nil { + if err = filepath.WalkDir(cpath, h.itemVisit); err != nil { return warnings, err } } @@ -424,7 +424,7 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) } - log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) + log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) } return warnings, nil @@ -440,8 +440,7 @@ func (h *Hub) LocalSync() ([]string, error) { return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) } - _, err = h.SyncDir(h.local.HubDir) - if err != nil { + if _, err = h.SyncDir(h.local.HubDir); err != nil { return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) } From 41d19de092fcdeed8971e667f1fc7d8fc0fd792c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:35:33 +0100 Subject: [PATCH 153/263] Refact cwhub (#2578) * Fix suggest functional tests * comments * non-empty SubItems() implies collections type * use "slices" from stdlib * No need to repeat author field in the index -- take it from the item key --- cmd/crowdsec-cli/config_restore.go | 1 + cmd/crowdsec-cli/require/require.go | 3 ++ pkg/cwhub/enable.go | 62 ++++++++++++++--------------- pkg/cwhub/hub.go | 15 +++---- pkg/cwhub/items.go | 2 +- pkg/cwhub/sync.go | 12 +----- test/bats/20_hub_collections.bats | 2 +- test/bats/20_hub_parsers.bats | 2 +- test/bats/20_hub_postoverflows.bats | 2 +- test/bats/20_hub_scenarios.bats | 2 +- 10 files changed, 46 insertions(+), 57 deletions(-) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 3c036684082..a8878f2ee02 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -22,6 +22,7 @@ type OldAPICfg struct { } // it's a rip of the cli version, but in silent-mode +// XXX: redundant, should call InstallItem func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) { var item = hub.GetItem(obtype, name) if item == nil { diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 855b27a7cee..43ab5adfed3 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient == nil { return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath) } + return nil } @@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error { if c.API.Server.OnlineClient.Credentials.PapiURL == "" { return fmt.Errorf("no PAPI URL in configuration") } + return nil } @@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error { if err := c.LoadDBConfig(); err != nil { return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err) } + return nil } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 38c547e0139..e769eb90cac 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -1,7 +1,6 @@ package cwhub -// Enable/disable items already installed (no downloading here) -// This file is not named install.go to avoid confusion with the functions in helpers.go +// Enable/disable items already downloaded import ( "fmt" @@ -11,8 +10,9 @@ import ( log "github.com/sirupsen/logrus" ) -// creates symlink between actual config file at hub.HubDir and hub.ConfigDir +// EnableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively +// XXX: called from config_restore otherwise no need to export func (h *Hub) EnableItem(target *Item) error { parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") @@ -42,16 +42,14 @@ func (h *Hub) EnableItem(target *Item) error { } // install sub-items if it's a collection - if target.Type == COLLECTIONS { - for _, sub := range target.SubItems() { - val, ok := h.Items[sub.Type][sub.Name] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) - } + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + } - if err := h.EnableItem(&val); err != nil { - return fmt.Errorf("while installing %s: %w", sub.Name, err) - } + if err := h.EnableItem(&val); err != nil { + return fmt.Errorf("while installing %s: %w", sub.Name, err) } } @@ -123,32 +121,30 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) } - // for a COLLECTIONS, disable sub-items - if target.Type == COLLECTIONS { - for _, sub := range target.SubItems() { - val, ok := h.Items[sub.Type][sub.Name] - if !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) - continue - } + // disable sub-items if any - it's a collection + for _, sub := range target.SubItems() { + val, ok := h.Items[sub.Type][sub.Name] + if !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) + continue + } - // check if the item doesn't belong to another collection before removing it - toRemove := true + // check if the item doesn't belong to another collection before removing it + toRemove := true - for _, collection := range val.BelongsToCollections { - if collection != target.Name { - toRemove = false - break - } + for _, collection := range val.BelongsToCollections { + if collection != target.Name { + toRemove = false + break } + } - if toRemove { - if err = h.DisableItem(&val, purge, force); err != nil { - return fmt.Errorf("while disabling %s: %w", sub.Name, err) - } - } else { - log.Infof("%s was not removed because it belongs to another collection", val.Name) + if toRemove { + if err = h.DisableItem(&val, purge, force); err != nil { + return fmt.Errorf("while disabling %s: %w", sub.Name, err) } + } else { + log.Infof("%s was not removed because it belongs to another collection", val.Name) } } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 1aaa0ca0e32..cf5fcb9e7e1 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -98,15 +98,17 @@ func ParseIndex(buff []byte) (HubItems, error) { for name, item := range RawIndex[itemType] { item.Name = name + + // if the item has no (redundant) author, take it from the json key + if item.Author == "" && strings.Contains(name, "/") { + item.Author = strings.Split(name, "/")[0] + } + item.Type = itemType x := strings.Split(item.RemotePath, "/") item.FileName = x[len(x)-1] RawIndex[itemType][name] = item - if itemType != COLLECTIONS { - continue - } - // if it's a collection, check its sub-items are present // XXX should be done later for _, sub := range item.SubItems() { @@ -130,11 +132,6 @@ func (h *Hub) ItemStats() []string { loaded := "" for _, itemType := range ItemTypes { - // ensure the order is always the same - if h.Items[itemType] == nil { - continue - } - if len(h.Items[itemType]) == 0 { continue } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index f5981216d4a..e49be1bfc62 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -16,7 +16,7 @@ const ( SCENARIOS = "scenarios" ) -// XXX: The order is important, as it is used to range over sub-items in collections +// The order is important, as it is used to range over sub-items in collections var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} type HubItems map[string]map[string]Item diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index c5d87613bf3..29d7dc85b34 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -11,6 +11,7 @@ import ( "strings" log "github.com/sirupsen/logrus" + "slices" ) func isYAMLFileName(path string) bool { @@ -255,7 +256,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { for _, version := range versions { if item.Versions[version].Digest != sha { - // log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v)) continue } @@ -365,15 +365,7 @@ func (h *Hub) CollectDepsCheck(v *Item) error { return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) } - skip := false - - for idx := range subItem.BelongsToCollections { - if subItem.BelongsToCollections[idx] == v.Name { - skip = true - } - } - - if !skip { + if !slices.Contains(subItem.BelongsToCollections, v.Name) { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 5c2d85317d2..423fe7bce30 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -132,7 +132,7 @@ teardown() { # autocorrect rune -1 cscli collections install crowdsecurity/ssshd - assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean crowdsecurity/sshd?" + assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean 'crowdsecurity/sshd'?" # install multiple rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index a505b0de9fb..05088cde96b 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -133,7 +133,7 @@ teardown() { # autocorrect rune -1 cscli parsers install crowdsecurity/sshd-logz - assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean crowdsecurity/sshd-logs?" + assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean 'crowdsecurity/sshd-logs'?" # install multiple rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index 9dda68361d5..b7768599329 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -132,7 +132,7 @@ teardown() { # autocorrect rune -1 cscli postoverflows install crowdsecurity/rdnf - assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean crowdsecurity/rdns?" + assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean 'crowdsecurity/rdns'?" # install multiple rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 29c73526e61..53b3f2078c7 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -133,7 +133,7 @@ teardown() { # autocorrect rune -1 cscli scenarios install crowdsecurity/ssh-tf - assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean crowdsecurity/ssh-bf?" + assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean 'crowdsecurity/ssh-bf'?" # install multiple rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf From bfd94ceda7588260aef8c6d9339b86e5cba3a6fe Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:27:33 +0100 Subject: [PATCH 154/263] make ParserIndex(), DownloadIndex() private methods (#2579) * unnecessary pointer type * ParseIndex() as hub method, don't collect missing items since they are never used * don't export hub.parseIndex(), hub.downloadIndex() --- pkg/cwhub/dataset.go | 4 +-- pkg/cwhub/enable.go | 1 + pkg/cwhub/errors.go | 2 +- pkg/cwhub/hub.go | 63 ++++++++++++++---------------------------- pkg/cwhub/hub_test.go | 6 ++-- pkg/cwhub/items.go | 1 + pkg/cwhub/remote.go | 4 +-- pkg/hubtest/hubtest.go | 17 +++++++----- test/bats/20_hub.bats | 8 ++++++ 9 files changed, 49 insertions(+), 57 deletions(-) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 802c5775c14..52b0675b084 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -15,7 +15,7 @@ import ( ) type DataSet struct { - Data []*types.DataSource `yaml:"data,omitempty"` + Data []types.DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -59,7 +59,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*types.DataSource, dataDir string) error { +func GetData(data []types.DataSource, dataDir string) error { for _, dataS := range data { destPath := filepath.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index e769eb90cac..fe8dcdeeabf 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -123,6 +123,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { // disable sub-items if any - it's a collection for _, sub := range target.SubItems() { + // XXX: we do this already when syncing, do we really need to do consistency checks here and there? val, ok := h.Items[sub.Type][sub.Name] if !ok { log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go index 7f39ace8ce0..3f7b87ac926 100644 --- a/pkg/cwhub/errors.go +++ b/pkg/cwhub/errors.go @@ -7,6 +7,6 @@ import ( var ( // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. // All attempts to download index or items will return this error. - ErrMissingReference = errors.New("Reference(s) missing in collection") ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") + ErrIndexNotFound = errors.New("index not found") ) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index cf5fcb9e7e1..460369f3474 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -2,7 +2,6 @@ package cwhub import ( "encoding/json" - "errors" "fmt" "os" "strings" @@ -20,10 +19,7 @@ type Hub struct { skippedTainted int } -var ( - theHub *Hub - ErrIndexNotFound = fmt.Errorf("index not found") -) +var theHub *Hub // GetHub returns the hub singleton // it returns an error if it's not initialized to avoid nil dereference @@ -44,59 +40,47 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo } if downloadIndex { - if err := remote.DownloadIndex(local.HubIndexFile); err != nil { + if err := remote.downloadIndex(local.HubIndexFile); err != nil { return nil, err } } log.Debugf("loading hub idx %s", local.HubIndexFile) - bidx, err := os.ReadFile(local.HubIndexFile) - if err != nil { - return nil, fmt.Errorf("unable to read index file: %w", err) - } - - ret, err := ParseIndex(bidx) - if err != nil { - if !errors.Is(err, ErrMissingReference) { - return nil, fmt.Errorf("failed to load index: %w", err) - } - - // XXX: why the error check if we bail out anyway? - return nil, err - } - theHub = &Hub{ - Items: ret, local: local, remote: remote, } - if _, err = theHub.LocalSync(); err != nil { + if err := theHub.parseIndex(); err != nil { + return nil, fmt.Errorf("failed to load index: %w", err) + } + + if _, err := theHub.LocalSync(); err != nil { return nil, fmt.Errorf("failed to sync hub index: %w", err) } return theHub, nil } -// ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections -func ParseIndex(buff []byte) (HubItems, error) { - var ( - RawIndex HubItems - missingItems []string - ) +// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections +func (h *Hub) parseIndex() error { + bidx, err := os.ReadFile(h.local.HubIndexFile) + if err != nil { + return fmt.Errorf("unable to read index file: %w", err) + } - if err := json.Unmarshal(buff, &RawIndex); err != nil { - return nil, fmt.Errorf("failed to unmarshal index: %w", err) + if err := json.Unmarshal(bidx, &h.Items); err != nil { + return fmt.Errorf("failed to unmarshal index: %w", err) } log.Debugf("%d item types in hub index", len(ItemTypes)) // Iterate over the different types to complete the struct for _, itemType := range ItemTypes { - log.Tracef("%s: %d items", itemType, len(RawIndex[itemType])) + log.Tracef("%s: %d items", itemType, len(h.Items[itemType])) - for name, item := range RawIndex[itemType] { + for name, item := range h.Items[itemType] { item.Name = name // if the item has no (redundant) author, take it from the json key @@ -107,24 +91,19 @@ func ParseIndex(buff []byte) (HubItems, error) { item.Type = itemType x := strings.Split(item.RemotePath, "/") item.FileName = x[len(x)-1] - RawIndex[itemType][name] = item + h.Items[itemType][name] = item // if it's a collection, check its sub-items are present - // XXX should be done later + // XXX should be done later, maybe report all missing at once? for _, sub := range item.SubItems() { - if _, ok := RawIndex[sub.Type][sub.Name]; !ok { + if _, ok := h.Items[sub.Type][sub.Name]; !ok { log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name) - missingItems = append(missingItems, sub.Name) } } } } - if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) - } - - return RawIndex, nil + return nil } // ItemStats returns total counts of the hub items diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 779bc25425a..21bb2d1fefd 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -45,7 +45,7 @@ func TestDownloadIndex(t *testing.T) { IndexPath: "", } - err = hub.remote.DownloadIndex(tmpIndex.Name()) + err = hub.remote.downloadIndex(tmpIndex.Name()) cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain @@ -57,7 +57,7 @@ func TestDownloadIndex(t *testing.T) { IndexPath: ".index.json", } - err = hub.remote.DownloadIndex(tmpIndex.Name()) + err = hub.remote.downloadIndex(tmpIndex.Name()) require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -71,6 +71,6 @@ func TestDownloadIndex(t *testing.T) { IndexPath: ".index.json", } - err = hub.remote.DownloadIndex("/does/not/exist/index.json") + err = hub.remote.downloadIndex("/does/not/exist/index.json") cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index e49be1bfc62..111cf771df9 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -234,6 +234,7 @@ func (h *Hub) AddItem(item Item) error { } } + // XXX: can this happen? return fmt.Errorf("ItemType %s is unknown", item.Type) } diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 8117cd1a64d..1bec6be98b8 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -29,8 +29,8 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil } -// DownloadIndex downloads the latest version of the index -func (r *RemoteHubCfg) DownloadIndex(localPath string) error { +// downloadIndex downloads the latest version of the index +func (r *RemoteHubCfg) downloadIndex(localPath string) error { if r == nil { return ErrNilRemoteHub } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index f8d9dce5f12..b110369ff96 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -56,15 +57,17 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } hubIndexFile := filepath.Join(hubPath, ".index.json") - bidx, err := os.ReadFile(hubIndexFile) - if err != nil { - return HubTest{}, fmt.Errorf("unable to read index file: %s", err) + + local := &csconfig.LocalHubCfg{ + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, + InstallDataDir: HubTestPath, } - // load hub index - hubIndex, err := cwhub.ParseIndex(bidx) + hub, err := cwhub.NewHub(local, nil, false) if err != nil { - return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err) + return HubTest{}, fmt.Errorf("unable to load hub: %s", err) } templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile) @@ -80,7 +83,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &cwhub.Hub{Items: hubIndex}, + HubIndex: hub, }, nil } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 1dc4fd87212..f6c1c87cfee 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -62,6 +62,14 @@ teardown() { assert_output --partial 'crowdsecurity/linux' } +@test "missing reference in hub index" { + new_hub=$(jq <"$HUB_DIR/.index.json" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') + echo "$new_hub" >"$HUB_DIR/.index.json" + rune -0 cscli hub list --error + assert_stderr --partial "Referred parsers crowdsecurity/smb-logs in collection crowdsecurity/smb doesn't exist." + assert_stderr --partial "Referred scenarios crowdsecurity/mysql-bf in collection crowdsecurity/mysql doesn't exist." +} + @test "cscli hub update" { #XXX: todo : From ad54b99bf98f2ef8a790fef56371970847c98565 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:02:02 +0100 Subject: [PATCH 155/263] Refact pkg/hubtest (#2580) * pkg/hubtest: lint (whitespace, empty lines) * use existing function to sort keys * lint * cscli hubtest: set TZ=UTC * dedup Coverage struct * pre-compile regexps * remove redundant type declarations or global vars --- cmd/crowdsec-cli/hubtest.go | 64 ++++++------ cmd/crowdsec-cli/hubtest_table.go | 10 +- pkg/hubtest/coverage.go | 137 ++++++++++++++----------- pkg/hubtest/hubtest.go | 25 +++-- pkg/hubtest/hubtest_item.go | 58 ++++++++--- pkg/hubtest/parser_assert.go | 165 +++++++++++++++++++++--------- pkg/hubtest/regexp.go | 11 ++ pkg/hubtest/scenario_assert.go | 57 ++++++++--- pkg/hubtest/utils.go | 6 ++ pkg/hubtest/utils_test.go | 18 ++-- 10 files changed, 359 insertions(+), 192 deletions(-) create mode 100644 pkg/hubtest/regexp.go diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 97bb8c8dd65..5052c133292 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -18,9 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -var ( - HubTest hubtest.HubTest -) +var HubTest hubtest.HubTest func NewHubTestCmd() *cobra.Command { var hubPath string @@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command { return nil }, } + cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") @@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command { return cmdHubTest } - func NewHubTestCreateCmd() *cobra.Command { parsers := []string{} postoverflows := []string{} @@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return nil }, } + cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test") cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test") cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test") @@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return cmdHubTestCreate } - func NewHubTestRunCmd() *cobra.Command { var noClean bool var runAll bool @@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { if !runAll && len(args) == 0 { printHelp(cmd) - return fmt.Errorf("Please provide test to run or --all flag") + return fmt.Errorf("please provide test to run or --all flag") } if runAll { @@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command { } } + // set timezone to avoid DST issues + os.Setenv("TZ", "UTC") + for _, test := range HubTest.Tests { if csConfig.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) @@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command { } } } - if csConfig.Cscli.Output == "human" { + + switch csConfig.Cscli.Output { + case "human": hubTestResultTable(color.Output, testResult) - } else if csConfig.Cscli.Output == "json" { + case "json": jsonResult := make(map[string][]string, 0) jsonResult["success"] = make([]string, 0) jsonResult["fail"] = make([]string, 0) @@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command { return fmt.Errorf("unable to json test result: %s", err) } fmt.Println(string(jsonStr)) + default: + return fmt.Errorf("only human/json output modes are supported") } if !success { @@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command { return nil }, } + cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed") cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests") @@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command { return cmdHubTestRun } - func NewHubTestCleanCmd() *cobra.Command { var cmdHubTestClean = &cobra.Command{ Use: "clean", @@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command { return cmdHubTestClean } - func NewHubTestInfoCmd() *cobra.Command { var cmdHubTestInfo = &cobra.Command{ Use: "info", @@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command { return cmdHubTestInfo } - func NewHubTestListCmd() *cobra.Command { var cmdHubTestList = &cobra.Command{ Use: "list", @@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command { return cmdHubTestList } - func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool @@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command { return fmt.Errorf("unable to load all tests: %+v", err) } var err error - scenarioCoverage := []hubtest.ScenarioCoverage{} - parserCoverage := []hubtest.ParserCoverage{} + scenarioCoverage := []hubtest.Coverage{} + parserCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 @@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command { parserTested := 0 for _, test := range parserCoverage { if test.TestsCount > 0 { - parserTested += 1 + parserTested++ } } parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100))) @@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command { if err != nil { return fmt.Errorf("while getting scenario coverage: %s", err) } + scenarioTested := 0 for _, test := range scenarioCoverage { if test.TestsCount > 0 { - scenarioTested += 1 + scenarioTested++ } } + scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } @@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command { os.Exit(0) } - if csConfig.Cscli.Output == "human" { + switch csConfig.Cscli.Output { + case "human": if showParserCov || showAll { hubTestParserCoverageTable(color.Output, parserCoverage) } @@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } - } else if csConfig.Cscli.Output == "json" { + case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") if err != nil { return err @@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) - } else { + default: return fmt.Errorf("only human/json output modes are supported") } return nil }, } + cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") @@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command { return cmdHubTestCoverage } - func NewHubTestEvalCmd() *cobra.Command { var evalExpression string var cmdHubTestEval = &cobra.Command{ @@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command { if err != nil { return fmt.Errorf("can't load test: %+v", err) } + err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err) } + output, err := test.ParserAssert.EvalExpression(evalExpression) if err != nil { return err } + fmt.Print(output) } return nil }, } + cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval") return cmdHubTestEval } - func NewHubTestExplainCmd() *cobra.Command { var cmdHubTestExplain = &cobra.Command{ Use: "explain", @@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command { } err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ParserAssert.LoadTest(test.ParserResultFile) - if err != nil { + + if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil { return fmt.Errorf("unable to load parser result after run: %s", err) } } err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) - if err != nil { + + if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil { return fmt.Errorf("unable to load scenario result after run: %s", err) } } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 9f28c36992d..9b31a79a20f 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) { t.Render() } -func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) { +func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Parser", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() } -func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) { +func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Scenario", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 29db527150e..5d3b79fe11e 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -5,173 +5,194 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "sort" "strings" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -type ParserCoverage struct { - Parser string +type Coverage struct { + Name string TestsCount int PresentIn map[string]bool //poorman's set } -type ScenarioCoverage struct { - Scenario string - TestsCount int - PresentIn map[string]bool -} - -func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { - var coverage []ParserCoverage +func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { - return coverage, fmt.Errorf("no parsers in hub index") + return nil, fmt.Errorf("no parsers in hub index") } - //populate from hub, iterate in alphabetical order - var pkeys []string - for pname := range h.HubIndex.Items[cwhub.PARSERS] { - pkeys = append(pkeys, pname) - } - sort.Strings(pkeys) - for _, pname := range pkeys { - coverage = append(coverage, ParserCoverage{ - Parser: pname, + + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, TestsCount: 0, PresentIn: make(map[string]bool), - }) + } } - //parser the expressions a-la-oneagain + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/parser.assert") if err != nil { - return coverage, fmt.Errorf("while find parser asserts : %s", err) + return nil, fmt.Errorf("while find parser asserts : %s", err) } + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + + match := parserResultRE.FindStringSubmatch(line) if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("parser") + + sidx := parserResultRE.SubexpIndex("parser") capturedParser := match[sidx] + for idx, pcover := range coverage { - if pcover.Parser == capturedParser { + if pcover.Name == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - parserNameSplit := strings.Split(pcover.Parser, "/") + + parserNameSplit := strings.Split(pcover.Name, "/") parserNameOnly := parserNameSplit[len(parserNameSplit)-1] + if parserNameOnly == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + capturedParserSplit := strings.Split(capturedParser, "/") capturedParserName := capturedParserSplit[len(capturedParserSplit)-1] + if capturedParserName == parserNameOnly { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if capturedParserName == parserNameOnly+"-logs" { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } + file.Close() } + return coverage, nil } -func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { - var coverage []ScenarioCoverage +func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { - return coverage, fmt.Errorf("no scenarios in hub index") - } - //populate from hub, iterate in alphabetical order - var pkeys []string - for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] { - pkeys = append(pkeys, scenarioName) + return nil, fmt.Errorf("no scenarios in hub index") } - sort.Strings(pkeys) - for _, scenarioName := range pkeys { - coverage = append(coverage, ScenarioCoverage{ - Scenario: scenarioName, + + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, TestsCount: 0, PresentIn: make(map[string]bool), - }) + } } - //parser the expressions a-la-oneagain + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/scenario.assert") if err != nil { - return coverage, fmt.Errorf("while find scenario asserts : %s", err) + return nil, fmt.Errorf("while find scenario asserts : %s", err) } + + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + match := scenarioResultRE.FindStringSubmatch(line) + if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("scenario") - scanner_name := match[sidx] + + sidx := scenarioResultRE.SubexpIndex("scenario") + scannerName := match[sidx] + for idx, pcover := range coverage { - if pcover.Scenario == scanner_name { + if pcover.Name == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - scenarioNameSplit := strings.Split(pcover.Scenario, "/") + + scenarioNameSplit := strings.Split(pcover.Name, "/") scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1] - if scenarioNameOnly == scanner_name { + + if scenarioNameOnly == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing") - fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing") + + fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing") + fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing") + if fixedProbingWord == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name { + + if fmt.Sprintf("%s-detection", pcover.Name) == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } file.Close() } + return coverage, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index b110369ff96..ec1f6ee5efe 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -30,28 +30,28 @@ const ( ) func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { - var err error - - hubPath, err = filepath.Abs(hubPath) + hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) } + // we can't use hubtest without the hub - if _, err := os.Stat(hubPath); os.IsNotExist(err) { + if _, err = os.Stat(hubPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) } + HubTestPath := filepath.Join(hubPath, "./.tests/") // we can't use hubtest without crowdsec binary - if _, err := exec.LookPath(crowdsecPath); err != nil { - if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) { + if _, err = exec.LookPath(crowdsecPath); err != nil { + if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath) } } // we can't use hubtest without cscli binary - if _, err := exec.LookPath(cscliPath); err != nil { - if _, err := os.Stat(cscliPath); os.IsNotExist(err) { + if _, err = exec.LookPath(cscliPath); err != nil { + if _, err = os.Stat(cscliPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath) } } @@ -59,9 +59,9 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, hubIndexFile := filepath.Join(hubPath, ".index.json") local := &csconfig.LocalHubCfg{ - HubDir: hubPath, - HubIndexFile: hubIndexFile, - InstallDir: HubTestPath, + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, InstallDataDir: HubTestPath, } @@ -89,10 +89,12 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { HubTestItem := &HubTestItem{} + testItem, err := NewTest(name, h) if err != nil { return HubTestItem, err } + h.Tests = append(h.Tests, testItem) return testItem, nil @@ -111,5 +113,6 @@ func (h *HubTest) LoadAllTests() error { } } } + return nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 44018a14acb..329e74a5ddc 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -7,11 +7,12 @@ import ( "path/filepath" "strings" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) type HubTestItemConfig struct { @@ -76,8 +77,6 @@ const ( BucketPourResultFileName = "bucketpour-dump.yaml" ) -var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns") - func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { testPath := filepath.Join(hubTest.HubTestPath, name) runtimeFolder := filepath.Join(testPath, "runtime") @@ -87,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { // read test configuration file configFileData := &HubTestItemConfig{} + yamlFile, err := os.ReadFile(configFilePath) if err != nil { log.Printf("no config file found in '%s': %v", testPath, err) } + err = yaml.Unmarshal(yamlFile, configFileData) if err != nil { return nil, fmt.Errorf("unmarshal: %v", err) @@ -101,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) + return &HubTestItem{ Name: name, Path: testPath, @@ -143,23 +145,25 @@ func (t *HubTestItem) InstallHub() error { if parser == "" { continue } - var parserDirDest string + if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) } + parserFileName := filepath.Base(parserSource) // runtime/hub/parsers/s00-raw/crowdsecurity/ hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath)) // runtime/parsers/s00-raw/ - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err) } + if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) } @@ -200,7 +204,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath) } - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) @@ -227,23 +231,25 @@ func (t *HubTestItem) InstallHub() error { if scenario == "" { continue } - var scenarioDirDest string + if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) } + scenarioFileName := filepath.Base(scenarioSource) // runtime/hub/scenarios/crowdsecurity/ hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath)) // runtime/parsers/scenarios/ - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err) } + if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -271,7 +277,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario) } - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -296,23 +302,25 @@ func (t *HubTestItem) InstallHub() error { if postoverflow == "" { continue } - var postoverflowDirDest string + if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) } + postoverflowFileName := filepath.Base(postoverflowSource) // runtime/hub/postoverflows/s00-enrich/crowdsecurity/ hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath)) // runtime/postoverflows/s00-enrich - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err) } + if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) } @@ -353,7 +361,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath) } - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) @@ -380,10 +388,12 @@ func (t *HubTestItem) InstallHub() error { Filter: "1==1", Statics: t.Config.OverrideStatics, } + b, err := yaml.Marshal(n) if err != nil { return fmt.Errorf("unable to marshal overrides: %s", err) } + tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath) if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil { return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err) @@ -403,6 +413,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } + log.Debugf("parser '%s' installed successfully in runtime environment", parserName) } } @@ -414,6 +425,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } + log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) } } @@ -425,6 +437,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } + log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) } } @@ -489,6 +502,8 @@ func (t *HubTestItem) Run() error { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } + crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + // copy template patterns folder to runtime folder if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) @@ -511,6 +526,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("unable to stat log file '%s': %s", logFile, err) } + if logFileStat.Size() == 0 { return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile) } @@ -518,6 +534,7 @@ func (t *HubTestItem) Run() error { cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) log.Debugf("%s", cscliRegisterCmd.String()) + output, err := cscliRegisterCmd.CombinedOutput() if err != nil { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { @@ -527,16 +544,20 @@ func (t *HubTestItem) Run() error { } cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"} + for labelKey, labelValue := range t.Config.Labels { arg := fmt.Sprintf("%s:%s", labelKey, labelValue) cmdArgs = append(cmdArgs, "-label", arg) } + crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) log.Debugf("%s", crowdsecCmd.String()) output, err = crowdsecCmd.CombinedOutput() + if log.GetLevel() >= log.DebugLevel || err != nil { fmt.Println(string(output)) } + if err != nil { return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) } @@ -553,8 +574,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + parserAssertFile.Close() } + assertFileStat, err := os.Stat(t.ParserAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err) @@ -565,6 +588,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ParserAssert.AutoGenAssertData = assertData t.ParserAssert.AutoGenAssert = true } else { @@ -576,12 +600,15 @@ func (t *HubTestItem) Run() error { // assert scenarios nbScenario := 0 + for _, scenario := range t.Config.Scenarios { if scenario == "" { continue } - nbScenario += 1 + + nbScenario++ } + if nbScenario > 0 { _, err := os.Stat(t.ScenarioAssert.File) if os.IsNotExist(err) { @@ -589,8 +616,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + scenarioAssertFile.Close() } + assertFileStat, err := os.Stat(t.ScenarioAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err) @@ -601,6 +630,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ScenarioAssert.AutoGenAssertData = assertData t.ScenarioAssert.AutoGenAssert = true } else { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index c9a18333620..620e2645e57 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -5,13 +5,11 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "time" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" "github.com/enescakir/emoji" "github.com/fatih/color" diff "github.com/r3labs/diff/v2" @@ -43,10 +41,10 @@ type ParserResult struct { Evt types.Event Success bool } + type ParserResults map[string]map[string][]ParserResult func NewParserAssert(file string) *ParserAssert { - ParserAssert := &ParserAssert{ File: file, NbAssert: 0, @@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert { AutoGenAssert: false, TestData: &ParserResults{}, } + return ParserAssert } @@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := p.AutoGenParserAssert() + return ret, nil } func (p *ParserAssert) LoadTest(filename string) error { - var err error parserDump, err := LoadParserDump(filename) if err != nil { return fmt.Errorf("loading parser dump file: %+v", err) } + p.TestData = parserDump + return nil } func (p *ParserAssert) AssertFile(testFile string) error { - file, err := os.Open(p.File) if err != nil { @@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error { if err := p.LoadTest(testFile); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := p.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - p.NbAssert += 1 + + p.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ =]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) variable := "" + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) variable = scanner.Text() } else { variable = match[1] } + result, err := p.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result p.Fails = append(p.Fails, *failedAssert) continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if p.NbAssert == 0 { assertData, err := p.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + p.AutoGenAssertData = assertData p.AutoGenAssert = true } + if len(p.Fails) == 0 { p.Success = true } @@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error { } func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *p.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { log.Errorf("failed to compile '%s' : %s", expression, err) return output, err } @@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return output, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) + if err != nil { return "", err } + return string(ret), nil } @@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) { func Escape(val string) string { val = strings.ReplaceAll(val, `\`, `\\`) val = strings.ReplaceAll(val, `"`, `\"`) + return val } func (p *ParserAssert) AutoGenParserAssert() string { //attempt to autogen parser asserts - var ret string + ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) + + //sort map keys for consistent order + stages := sortedMapKeys(*p.TestData) - //sort map keys for consistent ordre - var stages []string - for stage := range *p.TestData { - stages = append(stages, stage) - } - sort.Strings(stages) - ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) for _, stage := range stages { parsers := (*p.TestData)[stage] - //sort map keys for consistent ordre - var pnames []string - for pname := range parsers { - pnames = append(pnames, pname) - } - sort.Strings(pnames) + + //sort map keys for consistent order + pnames := sortedMapKeys(parsers) + for _, parser := range pnames { presults := parsers[parser] ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults)) + for pidx, result := range presults { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success) if !result.Success { continue } + for _, pkey := range sortedMapKeys(result.Evt.Parsed) { pval := result.Evt.Parsed[pkey] if pval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval)) } + for _, mkey := range sortedMapKeys(result.Evt.Meta) { mval := result.Evt.Meta[mkey] if mval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) } + for _, ekey := range sortedMapKeys(result.Evt.Enriched) { eval := result.Evt.Enriched[ekey] if eval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval)) } + for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) { uval := result.Evt.Unmarshaled[ukey] if uval == "" { continue } + base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey) + for _, line := range p.buildUnmarshaledAssert(base, uval) { ret += line } } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted) + if result.Evt.WhitelistReason != "" { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason)) } } } } + return ret } func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string { ret := make([]string, 0) + switch val := eval.(type) { case map[string]interface{}: for k, v := range val { @@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s default: log.Warningf("unknown type '%T' for key '%s'", val, ekey) } + return ret } func LoadParserDump(filepath string) (*ParserResults, error) { - var pdump ParserResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) { return nil, err } + pdump := ParserResults{} + if err := yaml.Unmarshal(results, &pdump); err != nil { return nil, err } /* we know that some variables should always be set, let's check if they're present in last parser output of last stage */ - stages := make([]string, 0, len(pdump)) - for k := range pdump { - stages = append(stages, k) - } - sort.Strings(stages) + + stages := sortedMapKeys(pdump) + var lastStage string + //Loop over stages to find last successful one with at least one parser for i := len(stages) - 2; i >= 0; i-- { if len(pdump[stages[i]]) != 0 { @@ -333,10 +361,13 @@ func LoadParserDump(filepath string) (*ParserResults, error) { break } } + parsers := make([]string, 0, len(pdump[lastStage])) + for k := range pdump[lastStage] { parsers = append(parsers, k) } + sort.Strings(parsers) lastParser := parsers[len(parsers)-1] @@ -357,47 +388,51 @@ type DumpOpts struct { ShowNotOkParsers bool } -func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) { +func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { //note : we can use line -> time as the unique identifier (of acquisition) - state := make(map[time.Time]map[string]map[string]ParserResult) assoc := make(map[time.Time]string, 0) - for stage, parsers := range parser_results { + for stage, parsers := range parserResults { for parser, results := range parsers { - for _, parser_res := range results { - evt := parser_res.Evt + for _, parserRes := range results { + evt := parserRes.Evt if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + if _, ok := state[evt.Line.Time][stage]; !ok { state[evt.Line.Time][stage] = make(map[string]ParserResult) } - state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success} - } + state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} + } } } - for bname, evtlist := range bucket_pour { + for bname, evtlist := range bucketPour { for _, evt := range evtlist { if evt.Line.Raw == "" { continue } + //it might be bucket overflow being reprocessed, skip this if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if _, ok := state[evt.Line.Time]["buckets"]; !ok { state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) } + state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} } } + yellow := color.New(color.FgYellow).SprintFunc() red := color.New(color.FgRed).SprintFunc() green := color.New(color.FgGreen).SprintFunc() @@ -409,19 +444,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum continue } } + fmt.Printf("line: %s\n", rawstr) + skeys := make([]string, 0, len(state[tstamp])) + for k := range state[tstamp] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "buckets" { continue } + skeys = append(skeys, k) } + sort.Strings(skeys) - //iterate stage - var prev_item types.Event + + // iterate stage + var prevItem types.Event for _, stage := range skeys { parsers := state[tstamp][stage] @@ -431,18 +472,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum fmt.Printf("\t%s %s\n", sep, stage) - pkeys := make([]string, 0, len(parsers)) - for k := range parsers { - pkeys = append(pkeys, k) - } - sort.Strings(pkeys) + pkeys := sortedMapKeys(parsers) for idx, parser := range pkeys { res := parsers[parser].Success sep := "ā”œ" + if idx == len(pkeys)-1 { sep = "ā””" } + created := 0 updated := 0 deleted := 0 @@ -451,16 +490,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum detailsDisplay := "" if res { - changelog, _ := diff.Diff(prev_item, parsers[parser].Evt) + changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) for _, change := range changelog { switch change.Type { case "create": created++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) case "update": detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) + if change.Path[0] == "Whitelisted" && change.To == true { whitelisted = true + if whitelistReason == "" { whitelistReason = parsers[parser].Evt.WhitelistReason } @@ -468,51 +510,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum updated++ case "delete": deleted++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) } } - prev_item = parsers[parser].Evt + + prevItem = parsers[parser].Evt } if created > 0 { changeStr += green(fmt.Sprintf("+%d", created)) } + if updated > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += yellow(fmt.Sprintf("~%d", updated)) } + if deleted > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += red(fmt.Sprintf("-%d", deleted)) } + if whitelisted { if len(changeStr) > 0 { changeStr += " " } + changeStr += red("[whitelisted]") } + if changeStr == "" { changeStr = yellow("unchanged") } + if res { fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) + if opts.Details { fmt.Print(detailsDisplay) } } else if opts.ShowNotOkParsers { fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) - } } } + sep := "ā””" + if len(state[tstamp]["buckets"]) > 0 { sep = "ā”œ" } + //did the event enter the bucket pour phase ? if _, ok := state[tstamp]["buckets"]["OK"]; ok { fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) @@ -521,27 +576,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum } else { fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) } + //now print bucket info if len(state[tstamp]["buckets"]) > 0 { fmt.Printf("\tā”œ Scenarios\n") } + bnames := make([]string, 0, len(state[tstamp]["buckets"])) + for k := range state[tstamp]["buckets"] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "OK" { continue } + bnames = append(bnames, k) } + sort.Strings(bnames) + for idx, bname := range bnames { sep := "ā”œ" if idx == len(bnames)-1 { sep = "ā””" } + fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) } + fmt.Println() } } diff --git a/pkg/hubtest/regexp.go b/pkg/hubtest/regexp.go new file mode 100644 index 00000000000..f9165eae3d1 --- /dev/null +++ b/pkg/hubtest/regexp.go @@ -0,0 +1,11 @@ +package hubtest + +import ( + "regexp" +) + +var ( + variableRE = regexp.MustCompile(`(?P[^ =]+) == .*`) + parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) + scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) +) diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index f5517c35056..011d3dcfb34 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -5,12 +5,10 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert { TestData: &BucketResults{}, PourData: &BucketPourInfo{}, } + return ScenarioAssert } @@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := s.AutoGenScenarioAssert() + return ret, nil } @@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) } + s.TestData = bucketDump if bucketpour != "" { @@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) } + s.PourData = pourDump } + return nil } @@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { if err := s.LoadTest(testFile, ""); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := s.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - s.NbAssert += 1 + + s.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ ]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) continue } + variable := match[1] + result, err := s.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result s.Fails = append(s.Fails, *failedAssert) + continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if s.NbAssert == 0 { assertData, err := s.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + s.AutoGenAssertData = assertData s.AutoGenAssert = true } @@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { } func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *s.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { @@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return nil, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) if err != nil { return "", err } + return string(ret), nil } @@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { } func (s *ScenarioAssert) AutoGenScenarioAssert() string { - //attempt to autogen parser asserts - var ret string - ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + // attempt to autogen scenario asserts + ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + for eventIndex, event := range *s.TestData { for ipSrc, source := range event.Overflow.Sources { ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex) @@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string { ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope) ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value) } + for evtIndex, evt := range event.Overflow.Alert.Events { for _, meta := range evt.Meta { ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) } } + ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario) ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation) ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount) } + return ret } @@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) { } func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { - var bucketDump BucketPourInfo - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { return nil, err } + var bucketDump BucketPourInfo + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } @@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { } func LoadScenarioDump(filepath string) (*BucketResults, error) { - var bucketDump BucketResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) { return nil, err } + var bucketDump BucketResults + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 489b324c4b3..090f1f85e3d 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string { for k := range m { keys = append(keys, k) } + sort.Strings(keys) + return keys } @@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error { } current := absSubPath + for { if current == absPath { return fmt.Errorf("cannot copy a folder onto itself") } + up := filepath.Dir(current) if current == up { break } + current = up } + return nil } diff --git a/pkg/hubtest/utils_test.go b/pkg/hubtest/utils_test.go index de4f1aac386..ce86785af9e 100644 --- a/pkg/hubtest/utils_test.go +++ b/pkg/hubtest/utils_test.go @@ -3,16 +3,16 @@ package hubtest import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCheckPathNotContained(t *testing.T) { - assert.Nil(t, checkPathNotContained("/foo", "/bar")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/foo")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/")) - assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) - assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) - assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar")) - assert.NotNil(t, checkPathNotContained("/", "/foo")) - assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz")) + require.NoError(t, checkPathNotContained("/foo", "/bar")) + require.NoError(t, checkPathNotContained("/foo/bar", "/foo")) + require.NoError(t, checkPathNotContained("/foo/bar", "/")) + require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) + require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) + require.Error(t, checkPathNotContained("/foo", "/foo/bar")) + require.Error(t, checkPathNotContained("/", "/foo")) + require.Error(t, checkPathNotContained("/", "/foo/bar/baz")) } From f4b5bcb865aaef1680f81f7ee1a64e206003438f Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:21:59 +0100 Subject: [PATCH 156/263] Refact cwhub: version comparison and branch selection (#2581) * simplify GetItemByPath * hub: sort version numbers by semver * replace golang.org/x/mod/semver with github.com/Masterminds/semver/v3 (would not compare correctly) * fix nil dereference with tainted items * update tests for collections, postoverflows * fix nil deref * don't fallback to master if hub is not found, improve message * explicit message for unknown version / tainted collections --- cmd/crowdsec-cli/hub.go | 12 +---- cmd/crowdsec-cli/item_metrics.go | 27 ++++++----- cmd/crowdsec-cli/items.go | 4 +- cmd/crowdsec-cli/require/require.go | 2 +- pkg/cwhub/errors.go | 11 ++++- pkg/cwhub/hub.go | 2 +- pkg/cwhub/items.go | 27 ++++++++++- pkg/cwhub/leakybucket.go | 13 ++---- pkg/cwhub/remote.go | 2 +- pkg/cwhub/sync.go | 43 +++++++++++++---- test/bats/20_hub_collections.bats | 31 +++++++------ test/bats/20_hub_items.bats | 72 +++++++++++++++++++++++++++++ test/bats/20_hub_postoverflows.bats | 38 ++++++++------- test/bats/20_hub_scenarios.bats | 4 +- 14 files changed, 207 insertions(+), 81 deletions(-) create mode 100644 test/bats/20_hub_items.bats diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 447462ad7dd..6d305441c1f 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -1,7 +1,6 @@ package main import ( - "errors" "fmt" "github.com/fatih/color" @@ -94,16 +93,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error { // don't use require.Hub because if there is no index file, it would fail hub, err := cwhub.NewHub(local, remote, true) if err != nil { - // XXX: this should be done when downloading items, too - // but what is the fallback to master actually solving? - if !errors.Is(err, cwhub.ErrIndexNotFound) { - return fmt.Errorf("failed to get Hub index: %w", err) - } - log.Warnf("Could not find index file for branch '%s', using 'master'", remote.Branch) - remote.Branch = "master" - if hub, err = cwhub.NewHub(local, remote, true); err != nil { - return fmt.Errorf("failed to get Hub index after retry: %w", err) - } + return fmt.Errorf("failed to update hub: %w", err) } // use LocalSync to get warnings about tainted / outdated items diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index 42759aa8fd4..4442a332e52 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -19,7 +19,7 @@ import ( ) // XXX: this should not need hub? -func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) { +func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) error { switch hubItem.Type { case cwhub.PARSERS: metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) @@ -28,24 +28,27 @@ func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) { metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) scenarioMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.COLLECTIONS: - for _, item := range hubItem.Parsers { - metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, item) - parserMetricsTable(color.Output, item, metrics) + for _, parserName := range hubItem.Parsers { + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, parserName) + parserMetricsTable(color.Output, parserName, metrics) } - for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, item) - scenarioMetricsTable(color.Output, item, metrics) + for _, scenarioName := range hubItem.Scenarios { + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, scenarioName) + scenarioMetricsTable(color.Output, scenarioName, metrics) } - for _, item := range hubItem.Collections { - hubItem = hub.GetItem(cwhub.COLLECTIONS, item) - if hubItem == nil { - log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) + for _, collName := range hubItem.Collections { + subColl := hub.GetItem(cwhub.COLLECTIONS, collName) + if subColl == nil { + return fmt.Errorf("unable to retrieve sub-collection '%s' from '%s'", collName, hubItem.Name) + } + if err := ShowMetrics(hub, subColl); err != nil { + return err } - ShowMetrics(hub, hubItem) } default: log.Errorf("item of type '%s' is unknown", hubItem.Type) } + return nil } // GetParserMetric is a complete rip from prom2json diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 9e13eb17937..6d913618363 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -157,7 +157,9 @@ func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error { if csConfig.Cscli.Output == "human" && showMetrics { fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hub, item) + if err := ShowMetrics(hub, item); err != nil { + return err + } } return nil diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 43ab5adfed3..47ad50d0aec 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -91,7 +91,7 @@ func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) { hub, err := cwhub.NewHub(local, remote, false) if err != nil { - return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) + return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } return hub, nil diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go index 3f7b87ac926..ec389921c7b 100644 --- a/pkg/cwhub/errors.go +++ b/pkg/cwhub/errors.go @@ -2,11 +2,20 @@ package cwhub import ( "errors" + "fmt" ) var ( // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. // All attempts to download index or items will return this error. ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") - ErrIndexNotFound = errors.New("index not found") ) + +type IndexNotFoundError struct { + URL string + Branch string +} + +func (e IndexNotFoundError) Error() string { + return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch) +} diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 460369f3474..823ede257c7 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -57,7 +57,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo } if _, err := theHub.LocalSync(); err != nil { - return nil, fmt.Errorf("failed to sync hub index: %w", err) + return nil, fmt.Errorf("failed to sync items: %w", err) } return theHub, nil diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 111cf771df9..399b98950cd 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" + "github.com/Masterminds/semver/v3" "github.com/enescakir/emoji" - "golang.org/x/mod/semver" ) const ( @@ -16,6 +16,13 @@ const ( SCENARIOS = "scenarios" ) +const ( + VersionUpToDate = iota + VersionUpdateAvailable + VersionUnknown + VersionFuture +) + // The order is important, as it is used to range over sub-items in collections var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} @@ -178,7 +185,23 @@ func (i *Item) Status() (string, emoji.Emoji) { // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { - return semver.Compare("v"+i.Version, "v"+i.LocalVersion) + local, err := semver.NewVersion(i.LocalVersion) + if err != nil { + return VersionUnknown + } + + // hub versions are already validated while syncing, ignore errors + latest, _ := semver.NewVersion(i.Version) + + if local.LessThan(latest) { + return VersionUpdateAvailable + } + + if local.Equal(latest) { + return VersionUpToDate + } + + return VersionFuture } // validPath returns true if the (relative) path is allowed for the item diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go index e57d6fa5a26..5d97153cbf8 100644 --- a/pkg/cwhub/leakybucket.go +++ b/pkg/cwhub/leakybucket.go @@ -38,23 +38,16 @@ func itemKey(itemPath string) (string, error) { } // GetItemByPath retrieves the item from the hub index based on its path. -// To achieve this it resolves a symlink to find the associated hub item. func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { itemKey, err := itemKey(itemPath) if err != nil { return nil, err } - // XXX: use GetItem() - m := h.GetItemMap(itemType) - if m == nil { - return nil, fmt.Errorf("item type %s doesn't exist", itemType) - } - - v, ok := m[itemKey] - if !ok { + item := h.GetItem(itemType, itemKey) + if item == nil { return nil, fmt.Errorf("%s not found in %s", itemKey, itemType) } - return &v, nil + return item, nil } diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 1bec6be98b8..91dc32db717 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -53,7 +53,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { - return ErrIndexNotFound + return IndexNotFoundError{req.URL.String(), r.Branch} } return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 29d7dc85b34..5665146fbb5 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -10,6 +10,7 @@ import ( "sort" "strings" + "github.com/Masterminds/semver/v3" log "github.com/sirupsen/logrus" "slices" ) @@ -32,7 +33,7 @@ func handleSymlink(path string) (string, error) { return "", fmt.Errorf("failed to unlink %s: %w", path, err) } - // XXX: is this correct? + // ignore this file return "", nil } @@ -125,6 +126,28 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { return ret, inhub, nil } +// sortedVersions returns the input data, sorted in reverse order by semver +func sortedVersions(raw []string) ([]string, error) { + vs := make([]*semver.Version, len(raw)) + for i, r := range raw { + v, err := semver.NewVersion(r) + if err != nil { + return nil, fmt.Errorf("%s: %w", r, err) + } + + vs[i] = v + } + + sort.Sort(sort.Reverse(semver.Collection(vs))) + + ret := make([]string, len(vs)) + for i, v := range vs { + ret[i] = v.Original() + } + + return ret, nil +} + func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { var ( local bool @@ -174,7 +197,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { log.Tracef("%s points to %s", path, hubpath) if hubpath == "" { - // XXX: is this correct? + // ignore this file return nil } } @@ -246,13 +269,15 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { } // let's reverse sort the versions to deal with hash collisions (#154) - // XXX: we sure, lexical sorting? versions := make([]string, 0, len(item.Versions)) for k := range item.Versions { versions = append(versions, k) } - sort.Sort(sort.Reverse(sort.StringSlice(versions))) + versions, err = sortedVersions(versions) + if err != nil { + return fmt.Errorf("while syncing %s %s: %w", info.ftype, info.fname, err) + } for _, version := range versions { if item.Versions[version].Digest != sha { @@ -315,7 +340,7 @@ func (h *Hub) CollectDepsCheck(v *Item) error { return nil } - if v.versionStatus() != 0 { // not up-to-date + if v.versionStatus() != VersionUpToDate { // not up-to-date log.Debugf("%s dependencies not checked: not up-to-date", v.Name) return nil } @@ -405,15 +430,17 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { vs := item.versionStatus() switch vs { - case 0: // latest + case VersionUpToDate: // latest if err := h.CollectDepsCheck(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) h.Items[COLLECTIONS][name] = item } - case 1: // not up-to-date + case VersionUpdateAvailable: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) - default: // version is higher than the highest available from hub? + case VersionFuture: warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) + case VersionUnknown: + warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) } log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 423fe7bce30..8452fc66680 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -79,26 +79,35 @@ teardown() { # XXX: check alphabetical order in human, json, raw } - @test "cscli collections list [collection]..." { + # non-existent + rune -1 cscli collections install foo/bar + assert_stderr --partial "can't find 'foo/bar' in collections" + + # not installed + rune -0 cscli collections list crowdsecurity/smb + assert_output --regexp 'crowdsecurity/smb.*disabled' + + # install two items rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb - # list one item + # list an installed item rune -0 cscli collections list crowdsecurity/sshd - assert_output --partial "crowdsecurity/sshd" + assert_output --regexp "crowdsecurity/sshd" refute_output --partial "crowdsecurity/smb" - # list multiple items - rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb + # list multiple installed and non installed items + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx assert_output --partial "crowdsecurity/sshd" assert_output --partial "crowdsecurity/smb" + assert_output --partial "crowdsecurity/nginx" rune -0 cscli collections list crowdsecurity/sshd -o json rune -0 jq '.collections | length' <(output) assert_output "1" - rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb crowdsecurity/nginx -o json rune -0 jq '.collections | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli collections list crowdsecurity/sshd -o raw rune -0 grep -vc 'name,status,version,description' <(output) @@ -108,14 +117,6 @@ teardown() { assert_output "2" } -@test "cscli collections list [collection]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli collections list crowdsecurity/sshd - # not existing - rune -1 cscli collections list blahblah/blahblah -} - @test "cscli collections install [collection]..." { rune -1 cscli collections install assert_stderr --partial 'requires at least 1 arg(s), only received 0' diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats new file mode 100644 index 00000000000..6ac19bf90b2 --- /dev/null +++ b/test/bats/20_hub_items.bats @@ -0,0 +1,72 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_purge_all + hub_strip_index +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- +# +# Tests that don't need to be repeated for each hub type +# + +@test "hub versions are correctly sorted during sync" { + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add two versions with the same hash, that don't sort the same way + # in a lexical vs semver sort. CrowdSec should report the latest version + + new_hub=$( \ + jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" \ + '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2":{"digest":$DIGEST, "deprecated": false}, "1.10": {"digest":$DIGEST, "deprecated": false}}}}}' \ + ) + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli collections install crowdsecurity/sshd + + truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml" + + rune -0 cscli collections inspect crowdsecurity/sshd -o json + # XXX: is this supposed to be tainted or up to date? + rune -0 jq -c '[.local_version,.up_to_date,.tainted]' <(output) + assert_json '["1.10",null,null]' +} + +@test "hub index with invalid (non semver) version numbers" { + new_hub=$( \ + jq <"$HUB_DIR/.index.json" \ + '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2.3.4":{"digest":"foo", "deprecated": false}}}}}' \ + ) + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli collections install crowdsecurity/sshd + + rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics + # XXX: we are on the verbose side here... + assert_stderr --partial "failed to read Hub index: failed to sync items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" +} + diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index b7768599329..7e3714de9d6 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -79,41 +79,43 @@ teardown() { # XXX: check alphabetical order in human, json, raw } - @test "cscli postoverflows list [scenario]..." { + # non-existent + rune -1 cscli postoverflows install foo/bar + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + + + # not installed + rune -0 cscli postoverflows list crowdsecurity/rdns + assert_output --regexp 'crowdsecurity/rdns.*disabled' + + # install two items rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist - # list one item + # list an installed item rune -0 cscli postoverflows list crowdsecurity/rdns - assert_output --partial "crowdsecurity/rdns" + assert_output --regexp "crowdsecurity/rdns.*enabled" refute_output --partial "crowdsecurity/cdn-whitelist" - # list multiple items - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist + # list multiple installed and non installed items + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range assert_output --partial "crowdsecurity/rdns" assert_output --partial "crowdsecurity/cdn-whitelist" + assert_output --partial "crowdsecurity/ipv6_to_range" rune -0 cscli postoverflows list crowdsecurity/rdns -o json rune -0 jq '.postoverflows | length' <(output) assert_output "1" - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o json rune -0 jq '.postoverflows | length' <(output) - assert_output "2" + assert_output "3" rune -0 cscli postoverflows list crowdsecurity/rdns -o raw rune -0 grep -vc 'name,status,version,description' <(output) assert_output "1" - rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist crowdsecurity/ipv6_to_range -o raw rune -0 grep -vc 'name,status,version,description' <(output) - assert_output "2" -} - -@test "cscli postoverflows list [scenario]... (not installed / not existing)" { - skip "not implemented yet" - # not installed - rune -1 cscli postoverflows list crowdsecurity/rdns - # not existing - rune -1 cscli postoverflows list blahblah/blahblah + assert_output "3" } @test "cscli postoverflows install [scenario]..." { @@ -157,6 +159,8 @@ teardown() { assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" } +# XXX: test install with --force +# XXX: test install with --ignore @test "cscli postoverflows inspect [scenario]..." { rune -1 cscli postoverflows inspect diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 53b3f2078c7..eb319382316 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -145,7 +145,6 @@ teardown() { assert_output --partial 'installed: true' } - @test "cscli scenarios install [scenario]... (file location and download-only)" { # simple install rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only @@ -159,6 +158,9 @@ teardown() { assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" } +# XXX: test install with --force +# XXX: test install with --ignore + @test "cscli scenarios inspect [scenario]..." { rune -1 cscli scenarios inspect From 152c9407740d47145a6aed09ae5365e6f60fe7aa Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 8 Nov 2023 20:24:44 +0100 Subject: [PATCH 157/263] wip --- cmd/crowdsec/crowdsec.go | 5 +++ pkg/acquisition/modules/waap/waap.go | 15 +++++-- pkg/waf/loader.go | 59 ++++++++++++++++++++++++++++ pkg/waf/waap.go | 28 ++++++++++++- pkg/waf/waap_rules_collection.go | 43 -------------------- 5 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 pkg/waf/loader.go diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 3b3a69cd33f..a2aaba97beb 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -18,6 +18,7 @@ import ( leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/crowdsec/pkg/waf" ) func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { @@ -38,6 +39,10 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { return nil, fmt.Errorf("while loading scenarios: %w", err) } + if err := waf.LoadWaapRules(); err != nil { + return nil, fmt.Errorf("while loading waap rules: %w", err) + } + if err := LoadAcquisition(cConfig); err != nil { return nil, fmt.Errorf("while loading acquisition config: %w", err) } diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 3089a2ce1e8..4fbacfeb57b 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -133,9 +133,18 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { //let's load the associated waap_config: if w.config.WaapConfigPath != "" { waapCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "waap_config")} - err := waapCfg.Load(w.config.WaapConfigPath) - if err != nil { - return fmt.Errorf("unable to load waap_config : %s", err) + if w.config.WaapConfigPath != "" { + err := waapCfg.LoadByPath(w.config.WaapConfigPath) + if err != nil { + return fmt.Errorf("unable to load waap_config : %s", err) + } + } else if w.config.WaapConfig != "" { + err := waapCfg.Load(w.config.WaapConfig) + if err != nil { + return fmt.Errorf("unable to load waap_config : %s", err) + } + } else { + return fmt.Errorf("no waap_config provided") } w.WaapRuntime, err = waapCfg.Build() if err != nil { diff --git a/pkg/waf/loader.go b/pkg/waf/loader.go new file mode 100644 index 00000000000..5acc136fc7b --- /dev/null +++ b/pkg/waf/loader.go @@ -0,0 +1,59 @@ +package waf + +import ( + "fmt" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +var waapRules map[string]WaapCollectionConfig = make(map[string]WaapCollectionConfig) //FIXME: would probably be better to have a struct for this + +func LoadWaapRules() error { + hub, err := cwhub.GetHub() + if err != nil { + return fmt.Errorf("unable to load hub : %s", err) + } + + for _, hubWafRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { + //log.Infof("loading %s", hubWafRuleItem.LocalPath) + if !hubWafRuleItem.Installed { + continue + } + + content, err := os.ReadFile(hubWafRuleItem.LocalPath) + + if err != nil { + log.Warnf("unable to read file %s : %s", hubWafRuleItem.LocalPath, err) + continue + } + + var rule WaapCollectionConfig + + err = yaml.UnmarshalStrict(content, &rule) + + if err != nil { + log.Warnf("unable to unmarshal file %s : %s", hubWafRuleItem.LocalPath, err) + continue + } + + if rule.Type != WAAP_RULE { + log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubWafRuleItem.LocalPath) + continue + } + + rule.hash = hubWafRuleItem.LocalHash + rule.version = hubWafRuleItem.Version + + log.Infof("Adding %s to waap rules", rule.Name) + + waapRules[rule.Name] = rule + } + + if len(waapRules) == 0 { + return fmt.Errorf("no waap rules found in hub") + } + return nil +} diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 9846b7c222b..96b585e59e8 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -7,6 +7,7 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -104,7 +105,7 @@ func (w *WaapRuntimeConfig) ClearResponse() { w.Response.SendEvent = true } -func (wc *WaapConfig) Load(file string) error { +func (wc *WaapConfig) LoadByPath(file string) error { wc.Logger.Debugf("loading config %s", file) @@ -147,6 +148,31 @@ func (wc *WaapConfig) Load(file string) error { return nil } +func (wc *WaapConfig) Load(configName string) error { + hub, err := cwhub.GetHub() + if err != nil { + return fmt.Errorf("unable to load hub : %s", err) + } + + waapConfigs := hub.GetItemMap(cwhub.WAAP_CONFIGS) + + for _, hubWaapConfigItem := range waapConfigs { + if !hubWaapConfigItem.Installed { + continue + } + if hubWaapConfigItem.Name != configName { + continue + } + wc.Logger.Infof("loading %s", hubWaapConfigItem.LocalPath) + err = wc.LoadByPath(hubWaapConfigItem.LocalPath) + if err != nil { + return fmt.Errorf("unable to load waap-config %s : %s", hubWaapConfigItem.LocalPath, err) + } + } + + return nil +} + func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret := &WaapRuntimeConfig{} ret.Name = wc.Name diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 45e5995b3f2..720be295fdb 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -9,7 +9,6 @@ import ( corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" - "gopkg.in/yaml.v2" log "github.com/sirupsen/logrus" ) @@ -50,53 +49,11 @@ var WaapRulesDetails = make(map[int]RulesDetails) func LoadCollection(collection string) (WaapCollection, error) { - //FIXME: do it once globally - waapRules := make(map[string]WaapCollectionConfig) - hub, err := cwhub.GetHub() if err != nil { return WaapCollection{}, fmt.Errorf("unable to load hub : %s", err) } - for _, hubWafRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { - //log.Infof("loading %s", hubWafRuleItem.LocalPath) - if !hubWafRuleItem.Installed { - continue - } - - content, err := os.ReadFile(hubWafRuleItem.LocalPath) - - if err != nil { - log.Warnf("unable to read file %s : %s", hubWafRuleItem.LocalPath, err) - continue - } - - var rule WaapCollectionConfig - - err = yaml.UnmarshalStrict(content, &rule) - - if err != nil { - log.Warnf("unable to unmarshal file %s : %s", hubWafRuleItem.LocalPath, err) - continue - } - - if rule.Type != WAAP_RULE { //FIXME: rename to waap-rule when hub is properly updated - log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubWafRuleItem.LocalPath) - continue - } - - rule.hash = hubWafRuleItem.LocalHash - rule.version = hubWafRuleItem.Version - - log.Infof("Adding %s to waap rules", rule.Name) - - waapRules[rule.Name] = rule - } - - if len(waapRules) == 0 { - return WaapCollection{}, fmt.Errorf("no waap rules found in hub") - } - var loadedRule WaapCollectionConfig var ok bool From 1154ada2df4ea7d139359f436f27916c7f640c01 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 8 Nov 2023 20:32:58 +0100 Subject: [PATCH 158/263] up --- cmd/crowdsec-cli/itemcommands.go | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 5a31dda2e36..a8b1050a9fe 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -124,6 +124,66 @@ cscli scenarios list -a cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, }, }, + "waap-rules": { + name: "waap-rules", + singular: "waap-rule", + oneOrMore: "waap-rule(s)", + help: cmdHelp{ + example: `cscli waap-rules list -a +cscli waap-rules install crowdsecurity/crs +cscli waap-rules inspect crowdsecurity/crs +cscli waap-rules upgrade crowdsecurity/crs +cscli waap-rules remove crowdsecurity/crs +`, + }, + installHelp: cmdHelp{ + example: `cscli waap-rules install crowdsecurity/crs`, + }, + removeHelp: cmdHelp{ + example: `cscli waap-rules remove crowdsecurity/crs`, + }, + upgradeHelp: cmdHelp{ + example: `cscli waap-rules upgrade crowdsecurity/crs`, + }, + inspectHelp: cmdHelp{ + example: `cscli waap-rules inspect crowdsecurity/crs`, + }, + listHelp: cmdHelp{ + example: `cscli waap-rules list +cscli waap-rules list -a +cscli waap-rules list crowdsecurity/crs`, + }, + }, + "waap-configs": { + name: "waap-configs", + singular: "waap-config", + oneOrMore: "waap-config(s)", + help: cmdHelp{ + example: `cscli waap-configs list -a +cscli waap-configs install crowdsecurity/vpatch +cscli waap-configs inspect crowdsecurity/vpatch +cscli waap-configs upgrade crowdsecurity/vpatch +cscli waap-configs remove crowdsecurity/vpatch +`, + }, + installHelp: cmdHelp{ + example: `cscli waap-configs install crowdsecurity/vpatch`, + }, + removeHelp: cmdHelp{ + example: `cscli waap-configs remove crowdsecurity/vpatch`, + }, + upgradeHelp: cmdHelp{ + example: `cscli waap-configs upgrade crowdsecurity/vpatch`, + }, + inspectHelp: cmdHelp{ + example: `cscli waap-configs inspect crowdsecurity/vpatch`, + }, + listHelp: cmdHelp{ + example: `cscli waap-configs list +cscli waap-configs list -a +cscli waap-configs list crowdsecurity/vpatch`, + }, + }, "collections": { name: "collections", singular: "collection", From 927310a4395c0840db3e23f01d06b4dce9d1a81d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 8 Nov 2023 20:37:05 +0100 Subject: [PATCH 159/263] up --- pkg/acquisition/modules/waap/waap.go | 29 ++++++++++++---------------- pkg/waf/waap.go | 3 ++- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 4fbacfeb57b..9561d895942 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -129,33 +129,28 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.InChan = make(chan waf.ParsedRequest) + waapCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "waap_config")} //let's load the associated waap_config: if w.config.WaapConfigPath != "" { - waapCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "waap_config")} - if w.config.WaapConfigPath != "" { - err := waapCfg.LoadByPath(w.config.WaapConfigPath) - if err != nil { - return fmt.Errorf("unable to load waap_config : %s", err) - } - } else if w.config.WaapConfig != "" { - err := waapCfg.Load(w.config.WaapConfig) - if err != nil { - return fmt.Errorf("unable to load waap_config : %s", err) - } - } else { - return fmt.Errorf("no waap_config provided") - } - w.WaapRuntime, err = waapCfg.Build() + err := waapCfg.LoadByPath(w.config.WaapConfigPath) if err != nil { - return fmt.Errorf("unable to build waap_config : %s", err) + return fmt.Errorf("unable to load waap_config : %s", err) } } else if w.config.WaapConfig != "" { - return fmt.Errorf("resolution of waap_config not implemented yet") + err := waapCfg.Load(w.config.WaapConfig) + if err != nil { + return fmt.Errorf("unable to load waap_config : %s", err) + } } else { return fmt.Errorf("no waap_config provided") } + w.WaapRuntime, err = waapCfg.Build() + if err != nil { + return fmt.Errorf("unable to build waap_config : %s", err) + } + err = w.WaapRuntime.ProcessOnLoadRules() if err != nil { diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 96b585e59e8..017e57ae268 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -168,9 +168,10 @@ func (wc *WaapConfig) Load(configName string) error { if err != nil { return fmt.Errorf("unable to load waap-config %s : %s", hubWaapConfigItem.LocalPath, err) } + return nil } - return nil + return fmt.Errorf("no waap-config found for %s", configName) } func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { From a0b0745f9d51fa7662c423a1563bf72f84834e97 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 8 Nov 2023 21:14:03 +0100 Subject: [PATCH 160/263] up --- pkg/waf/waap.go | 8 +-- pkg/waf/waap_rules_collection.go | 111 +++++++++++++++++-------------- 2 files changed, 64 insertions(+), 55 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 017e57ae268..7cede9df2ad 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -183,21 +183,21 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { //load rules for _, rule := range wc.OutOfBandRules { wc.Logger.Infof("loading outofband rule %s", rule) - collection, err := LoadCollection(rule) + collections, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) } - ret.OutOfBandRules = append(ret.OutOfBandRules, collection) + ret.OutOfBandRules = append(ret.OutOfBandRules, collections...) } wc.Logger.Infof("Loaded %d outofband rules", len(ret.OutOfBandRules)) for _, rule := range wc.InBandRules { wc.Logger.Infof("loading inband rule %s", rule) - collection, err := LoadCollection(rule) + collections, err := LoadCollection(rule) if err != nil { return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) } - ret.InBandRules = append(ret.InBandRules, collection) + ret.InBandRules = append(ret.InBandRules, collections...) } wc.Logger.Infof("Loaded %d inband rules", len(ret.InBandRules)) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 720be295fdb..7d96549fffd 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -47,77 +47,86 @@ type RulesDetails struct { // Is using the id is a good idea ? might be too specific to coraza and not easily reusable var WaapRulesDetails = make(map[int]RulesDetails) -func LoadCollection(collection string) (WaapCollection, error) { +func LoadCollection(pattern string) ([]WaapCollection, error) { hub, err := cwhub.GetHub() if err != nil { - return WaapCollection{}, fmt.Errorf("unable to load hub : %s", err) + return nil, fmt.Errorf("unable to load hub : %s", err) } - var loadedRule WaapCollectionConfig - var ok bool + ret := make([]WaapCollection, 0) - if loadedRule, ok = waapRules[collection]; !ok { - return WaapCollection{}, fmt.Errorf("no waap rules found for collection %s", collection) - } + for _, waapRule := range waapRules { - waapCol := WaapCollection{ - collectionName: loadedRule.Name, - } + matched, err := filepath.Match(pattern, waapRule.Name) - if loadedRule.SecLangFilesRules != nil { - for _, rulesFile := range loadedRule.SecLangFilesRules { - fullPath := filepath.Join(hub.GetDataDir(), rulesFile) - c, err := os.ReadFile(fullPath) - if err != nil { - log.Errorf("unable to read file %s : %s", rulesFile, err) - continue - } - for _, line := range strings.Split(string(c), "\n") { - if strings.HasPrefix(line, "#") { + if err != nil { + log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) + continue + } + + if !matched { + continue + } + + waapCol := WaapCollection{ + collectionName: waapRule.Name, + } + + if waapRule.SecLangFilesRules != nil { + for _, rulesFile := range waapRule.SecLangFilesRules { + fullPath := filepath.Join(hub.GetDataDir(), rulesFile) + c, err := os.ReadFile(fullPath) + if err != nil { + log.Errorf("unable to read file %s : %s", rulesFile, err) continue } - if strings.TrimSpace(line) == "" { - continue + for _, line := range strings.Split(string(c), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + waapCol.Rules = append(waapCol.Rules, line) } - waapCol.Rules = append(waapCol.Rules, line) } } - } - if loadedRule.SecLangRules != nil { - waapCol.Rules = append(waapCol.Rules, loadedRule.SecLangRules...) - } + if waapRule.SecLangRules != nil { + waapCol.Rules = append(waapCol.Rules, waapRule.SecLangRules...) + } - if loadedRule.Rules != nil { - for _, rule := range loadedRule.Rules { - strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, loadedRule.Name) - if err != nil { - log.Errorf("unable to convert rule %s : %s", rule.Name, err) - return WaapCollection{}, err - } - log.Infof("Adding rule %s", strRule) - waapCol.Rules = append(waapCol.Rules, strRule) - - //We only take the first id, as it's the one of the "main" rule - if _, ok := WaapRulesDetails[int(rulesId[0])]; !ok { - WaapRulesDetails[int(rulesId[0])] = RulesDetails{ - LogLevel: log.InfoLevel, - Hash: loadedRule.hash, - Version: loadedRule.version, - Name: loadedRule.Name, + if waapRule.Rules != nil { + for _, rule := range waapRule.Rules { + strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, waapRule.Name) + if err != nil { + log.Errorf("unable to convert rule %s : %s", rule.Name, err) + return nil, err + } + log.Infof("Adding rule %s", strRule) + waapCol.Rules = append(waapCol.Rules, strRule) + + //We only take the first id, as it's the one of the "main" rule + if _, ok := WaapRulesDetails[int(rulesId[0])]; !ok { + WaapRulesDetails[int(rulesId[0])] = RulesDetails{ + LogLevel: log.InfoLevel, + Hash: waapRule.hash, + Version: waapRule.version, + Name: waapRule.Name, + } + } else { + log.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) } - } else { - log.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) - } - for _, id := range rulesId { - SetRuleDebug(int(id), loadedRule.Debug) + for _, id := range rulesId { + SetRuleDebug(int(id), waapRule.Debug) + } } } + ret = append(ret, waapCol) } - - return waapCol, nil + return ret, nil } func (wcc WaapCollectionConfig) LoadCollection(collection string) (WaapCollection, error) { From ec4b5bdc8666838a31b4ecac6d0beb77dc4ec6c4 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:34:14 +0100 Subject: [PATCH 161/263] Refact cwhub (#2583) * no need to use NewRequest() * download error messages * cscli hub list: fix item stats * Method item.HasSubItems() - avoid explicit type check * cscli config restore: drop silent install, just call InstallItem * no backpointer yet --- cmd/crowdsec-cli/config_restore.go | 31 +----------------------------- cmd/crowdsec-cli/hub.go | 2 +- pkg/cwhub/dataset.go | 13 ++++--------- pkg/cwhub/enable.go | 4 ++-- pkg/cwhub/helpers.go | 18 +++++++---------- pkg/cwhub/helpers_test.go | 2 +- pkg/cwhub/items.go | 8 ++++++++ pkg/cwhub/remote.go | 11 +++-------- pkg/cwhub/sync.go | 27 ++++++++------------------ 9 files changed, 35 insertions(+), 81 deletions(-) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index a8878f2ee02..d33b2e61a3c 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -21,31 +21,6 @@ type OldAPICfg struct { Password string `json:"password"` } -// it's a rip of the cli version, but in silent-mode -// XXX: redundant, should call InstallItem -func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) { - var item = hub.GetItem(obtype, name) - if item == nil { - return "", fmt.Errorf("error retrieving item") - } - err := hub.DownloadLatest(item, false, false) - if err != nil { - return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) - } - if err = hub.AddItem(*item); err != nil { - return "", err - } - - err = hub.EnableItem(item) - if err != nil { - return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) - } - if err := hub.AddItem(*item); err != nil { - return "", err - } - return fmt.Sprintf("Enabled %s", item.Name), nil -} - func restoreHub(dirPath string) error { hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) if err != nil { @@ -70,13 +45,9 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - label, err := silentInstallItem(hub, toinstall, itype) + err := hub.InstallItem(toinstall, itype, false, false) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) - } else if label != "" { - log.Infof("Installed %s : %s", toinstall, label) - } else { - log.Printf("Installed %s : ok", toinstall) } } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 6d305441c1f..abf99175a64 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -59,7 +59,7 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(v) } - for line := range hub.ItemStats() { + for _, line := range hub.ItemStats() { log.Info(line) } diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 52b0675b084..d57827c3e11 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -21,24 +21,19 @@ type DataSet struct { func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) - req, err := http.NewRequest(http.MethodGet, url, nil) + resp, err := http.DefaultClient.Get(url) if err != nil { - return err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err + return fmt.Errorf("while downloading %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("got HTTP status '%s' from %s", resp.Status, url) + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(resp.Body) if err != nil { - return err + return fmt.Errorf("while downloading %s: %w", url, err) } file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index fe8dcdeeabf..f2281bb1fe9 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -27,7 +27,7 @@ func (h *Hub) EnableItem(target *Item) error { } // if it's a collection, check sub-items even if the collection file itself is up-to-date - if target.UpToDate && target.Type != COLLECTIONS { + if target.UpToDate && !target.HasSubItems() { log.Tracef("%s is installed and up-to-date, skip.", target.Name) return nil } @@ -41,7 +41,7 @@ func (h *Hub) EnableItem(target *Item) error { } } - // install sub-items if it's a collection + // install sub-items if any for _, sub := range target.SubItems() { val, ok := h.Items[sub.Type][sub.Name] if !ok { diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 0357259edc9..89e9c747cdf 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -158,12 +158,13 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro // XXX: should return the path of the downloaded file (taken from DownloadItem) log.Debugf("Downloading %s %s", target.Type, target.Name) - if target.Type != COLLECTIONS { + if !target.HasSubItems() { if !target.Installed && updateOnly && target.Downloaded { log.Debugf("skipping upgrade of %s: not installed", target.Name) return nil } + // XXX: return h.DownloadItem(target, overwrite) } @@ -182,7 +183,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) // recurse as it's a collection - if sub.Type == COLLECTIONS { + if sub.HasSubItems() { log.Tracef("collection, recurse") if err := h.DownloadLatest(&val, overwrite, updateOnly); err != nil { @@ -235,24 +236,19 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { } } - req, err := http.NewRequest(http.MethodGet, url, nil) + resp, err := http.DefaultClient.Get(url) if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) + return fmt.Errorf("while downloading %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("while reading %s: %w", req.URL.String(), err) + return fmt.Errorf("while downloading %s: %w", url, err) } hash := sha256.New() diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index a859c116d6f..15f31cdca41 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -180,7 +180,7 @@ func assertCollectionDepsInstalled(t *testing.T, collection string) { require.NoError(t, err) c := hub.Items[COLLECTIONS][collection] - require.NoError(t, hub.CollectDepsCheck(&c)) + require.NoError(t, hub.checkSubItems(&c)) } func pushUpdateToCollectionInHub() { diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 399b98950cd..53ccd541448 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -74,6 +74,14 @@ type SubItem struct { Name string } +func (i *Item) HasSubItems() bool { + return i.Type == COLLECTIONS +} + +func (i *SubItem) HasSubItems() bool { + return i.Type == COLLECTIONS +} + func (i *Item) IsLocal() bool { return i.Installed && !i.Downloaded } diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 91dc32db717..9e0c2c04db4 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -40,12 +40,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { return fmt.Errorf("failed to build hub index request: %w", err) } - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return fmt.Errorf("failed to build request for hub index: %w", err) - } - - resp, err := http.DefaultClient.Do(req) + resp, err := http.DefaultClient.Get(url) if err != nil { return fmt.Errorf("failed http request for hub index: %w", err) } @@ -53,10 +48,10 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { - return IndexNotFoundError{req.URL.String(), r.Branch} + return IndexNotFoundError{url, r.Branch} } - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String()) + return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(resp.Body) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 5665146fbb5..6193bf388b7 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -206,7 +206,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { if local && !inhub { log.Tracef("%s is a local file, skip", path) h.skippedLocal++ - // log.Infof("local scenario, skip.") _, fileName := filepath.Split(path) @@ -335,17 +334,18 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { return nil } -func (h *Hub) CollectDepsCheck(v *Item) error { - if v.Type != COLLECTIONS { +// checkSubItems checks for the presence, taint and version state of sub-items +func (h *Hub) checkSubItems(v *Item) error { + if !v.HasSubItems() { return nil } - if v.versionStatus() != VersionUpToDate { // not up-to-date + if v.versionStatus() != VersionUpToDate { log.Debugf("%s dependencies not checked: not up-to-date", v.Name) return nil } - // if it's a collection, ensure all the items are installed, or tag it as tainted + // ensure all the sub-items are installed, or tag the parent as tainted log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) for _, sub := range v.SubItems() { @@ -360,21 +360,10 @@ func (h *Hub) CollectDepsCheck(v *Item) error { continue } - if subItem.Type == COLLECTIONS { - log.Tracef("collec, recurse.") - - if err := h.CollectDepsCheck(&subItem); err != nil { - if subItem.Tainted { - v.Tainted = true - } - - return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) - } - - h.Items[sub.Type][sub.Name] = subItem + if err := h.checkSubItems(&subItem); err != nil { + return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - // propagate the state of sub-items to set if subItem.Tainted { v.Tainted = true return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) @@ -431,7 +420,7 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { vs := item.versionStatus() switch vs { case VersionUpToDate: // latest - if err := h.CollectDepsCheck(&item); err != nil { + if err := h.checkSubItems(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) h.Items[COLLECTIONS][name] = item } From f80d841188d57afaadfe73d405de3527b900cbfa Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:07:09 +0100 Subject: [PATCH 162/263] Refact cwhub: make some methods private (#2584) * make hub.enableItem() private * make hub.downloadLatest() private * make getData() private * make hub.disableItem() private * make hub.downloadItem() private * make hub.syncDir() private * make hub.localSync() private; keep warnings in Hub struct (no need to call LocalSync to get them) --- cmd/crowdsec-cli/hub.go | 8 ++------ pkg/cwhub/dataset.go | 4 ++-- pkg/cwhub/enable.go | 13 ++++++------- pkg/cwhub/enable_test.go | 26 +++++++++++++------------- pkg/cwhub/helpers.go | 30 +++++++++++++++--------------- pkg/cwhub/hub.go | 3 ++- pkg/cwhub/sync.go | 19 ++++++++++++------- 7 files changed, 52 insertions(+), 51 deletions(-) diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index abf99175a64..f934bd8561a 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -53,9 +53,7 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := hub.LocalSync() - for _, v := range warn { + for _, v := range hub.Warnings { log.Info(v) } @@ -96,9 +94,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to update hub: %w", err) } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := hub.LocalSync() - for _, v := range warn { + for _, v := range hub.Warnings { log.Info(v) } diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index d57827c3e11..2b3629a978d 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -54,7 +54,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []types.DataSource, dataDir string) error { +func getData(data []types.DataSource, dataDir string) error { for _, dataS := range data { destPath := filepath.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) @@ -91,7 +91,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } if download || force { - if err := GetData(data.Data, dataFolder); err != nil { + if err := getData(data.Data, dataFolder); err != nil { return fmt.Errorf("while getting data: %w", err) } } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index f2281bb1fe9..2f137fa76c8 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -10,10 +10,9 @@ import ( log "github.com/sirupsen/logrus" ) -// EnableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir +// enableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -// XXX: called from config_restore otherwise no need to export -func (h *Hub) EnableItem(target *Item) error { +func (h *Hub) enableItem(target *Item) error { parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") // create directories if needed @@ -48,7 +47,7 @@ func (h *Hub) EnableItem(target *Item) error { return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) } - if err := h.EnableItem(&val); err != nil { + if err := h.enableItem(&val); err != nil { return fmt.Errorf("while installing %s: %w", sub.Name, err) } } @@ -96,8 +95,8 @@ func (h *Hub) purgeItem(target Item) (Item, error) { return target, nil } -// DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { +// disableItem to disable an item managed by the hub, removes the symlink if purge is true +func (h *Hub) disableItem(target *Item, purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not var err error @@ -141,7 +140,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { } if toRemove { - if err = h.DisableItem(&val, purge, force); err != nil { + if err = h.disableItem(&val, purge, force); err != nil { return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } else { diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 8b7cac88db6..a476c76d758 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -10,20 +10,20 @@ import ( func testInstall(hub *Hub, t *testing.T, item Item) { // Install the parser - err := hub.DownloadLatest(&item, false, false) + err := hub.downloadLatest(&item, false, false) require.NoError(t, err, "failed to download %s", item.Name) - _, err = hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - err = hub.EnableItem(&item) + err = hub.enableItem(&item) require.NoError(t, err, "failed to enable %s", item.Name) - _, err = hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) @@ -41,7 +41,7 @@ func testTaint(hub *Hub, t *testing.T, item Item) { require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) // Local sync and check status - _, err = hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) @@ -51,11 +51,11 @@ func testUpdate(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := hub.DownloadLatest(&item, true, true) + err := hub.downloadLatest(&item, true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status - _, err = hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) @@ -66,26 +66,26 @@ func testDisable(hub *Hub, t *testing.T, item Item) { assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove - err := hub.DisableItem(&item, false, false) + err := hub.disableItem(&item, false, false) require.NoError(t, err, "failed to disable %s", item.Name) // Local sync and check status - warns, err := hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings: %+v", warns) + require.Empty(t, hub.Warnings) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge - err = hub.DisableItem(&item, true, false) + err = hub.disableItem(&item, true, false) require.NoError(t, err, "failed to purge %s", item.Name) // Local sync and check status - warns, err = hub.LocalSync() + err = hub.localSync() require.NoError(t, err, "failed to run localSync") - require.Empty(t, warns, "unexpected warnings: %+v", warns) + require.Empty(t, hub.Warnings) assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 89e9c747cdf..e8b49b81465 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -35,7 +35,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } // XXX: confusing semantic between force and updateOnly? - if err := h.DownloadLatest(item, force, true); err != nil { + if err := h.downloadLatest(item, force, true); err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } @@ -44,14 +44,14 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } if downloadOnly { - // XXX: should get the path from DownloadLatest + // XXX: should get the path from downloadLatest log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath)) return nil } // XXX: should we stop here if the item is already installed? - if err := h.EnableItem(item); err != nil { + if err := h.enableItem(item); err != nil { return fmt.Errorf("while enabling %s: %w", item.Name, err) } @@ -83,11 +83,11 @@ func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction b return false, nil } - if err := h.DisableItem(item, purge, forceAction); err != nil { + if err := h.disableItem(item, purge, forceAction); err != nil { return false, fmt.Errorf("unable to disable %s: %w", item.Name, err) } - // XXX: should take the value from DisableItem + // XXX: should take the value from disableItem removed = true if err := h.AddItem(*item); err != nil { @@ -127,7 +127,7 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error } } - if err := h.DownloadLatest(item, force, true); err != nil { + if err := h.downloadLatest(item, force, true); err != nil { return false, fmt.Errorf("%s: download failed: %w", item.Name, err) } @@ -153,9 +153,9 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error return updated, nil } -// DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { - // XXX: should return the path of the downloaded file (taken from DownloadItem) +// downloadLatest will download the latest version of Item to the tdir directory +func (h *Hub) downloadLatest(target *Item, overwrite bool, updateOnly bool) error { + // XXX: should return the path of the downloaded file (taken from downloadItem) log.Debugf("Downloading %s %s", target.Type, target.Name) if !target.HasSubItems() { @@ -165,7 +165,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro } // XXX: - return h.DownloadItem(target, overwrite) + return h.downloadItem(target, overwrite) } // collection @@ -186,21 +186,21 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro if sub.HasSubItems() { log.Tracef("collection, recurse") - if err := h.DownloadLatest(&val, overwrite, updateOnly); err != nil { + if err := h.downloadLatest(&val, overwrite, updateOnly); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } } downloaded := val.Downloaded - if err := h.DownloadItem(&val, overwrite); err != nil { + if err := h.downloadItem(&val, overwrite); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { - if err := h.EnableItem(&val); err != nil { + if err := h.enableItem(&val); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } @@ -208,14 +208,14 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro h.Items[sub.Type][sub.Name] = val } - if err := h.DownloadItem(target, overwrite); err != nil { + if err := h.downloadItem(target, overwrite); err != nil { return fmt.Errorf("failed to download item: %w", err) } return nil } -func (h *Hub) DownloadItem(target *Item, overwrite bool) error { +func (h *Hub) downloadItem(target *Item, overwrite bool) error { url, err := h.remote.urlTo(target.RemotePath) if err != nil { return fmt.Errorf("failed to build hub item request: %w", err) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 823ede257c7..f2b6c4b880b 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -17,6 +17,7 @@ type Hub struct { remote *RemoteHubCfg skippedLocal int skippedTainted int + Warnings []string } var theHub *Hub @@ -56,7 +57,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo return nil, fmt.Errorf("failed to load index: %w", err) } - if _, err := theHub.LocalSync(); err != nil { + if err := theHub.localSync(); err != nil { return nil, fmt.Errorf("failed to sync items: %w", err) } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 6193bf388b7..e9ad1b41c75 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -391,7 +391,7 @@ func (h *Hub) checkSubItems(v *Item) error { return nil } -func (h *Hub) SyncDir(dir string) ([]string, error) { +func (h *Hub) syncDir(dir string) ([]string, error) { warnings := []string{} // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last @@ -439,18 +439,23 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { } // Updates the info from HubInit() with the local state -func (h *Hub) LocalSync() ([]string, error) { +func (h *Hub) localSync() error { h.skippedLocal = 0 h.skippedTainted = 0 + h.Warnings = []string{} - warnings, err := h.SyncDir(h.local.InstallDir) + warnings, err := h.syncDir(h.local.InstallDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) } - if _, err = h.SyncDir(h.local.HubDir); err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + h.Warnings = append(h.Warnings, warnings...) + + if warnings, err = h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) } - return warnings, nil + h.Warnings = append(h.Warnings, warnings...) + + return nil } From ab8de1950668d7649a410e78c074ebbd725e2f46 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:19:38 +0100 Subject: [PATCH 163/263] Refact cwhub: move methods from hub to item (#2585) * Add back pointer Item.hub * Hub.enableItem() -> Item.enable() * Rename variable i -> idx (i is used for item instances) * Move Hub.purgeItem() -> Item.purge() * Move Hub.disableItem() -> Item.disable() * Move Hub.downloadItem() -> Item.download() * Move Hub.downloadLatest() -> Item.downloadLatest() * Move Hub.DownloadDataIfNeeded() -> Item.DownloadDataIfNeeded() * Move Hub.InstallItem() -> Item.Install() * Move Hub.RemoveItem() -> Item.Remove() * Move Hub.UpgradeItem() -> Item.Upgrade() * store hub items as pointers * No need to re-add items to the hub if we use pointers * Fix parameter calling order + regression test --- cmd/crowdsec-cli/config_restore.go | 7 +- cmd/crowdsec-cli/hub.go | 2 +- cmd/crowdsec-cli/itemcommands.go | 53 ++++---- pkg/cwhub/enable.go | 103 ++++++++------- pkg/cwhub/enable_test.go | 18 +-- pkg/cwhub/helpers.go | 199 ++++++++++++----------------- pkg/cwhub/helpers_test.go | 29 +++-- pkg/cwhub/hub.go | 2 +- pkg/cwhub/items.go | 42 ++---- pkg/cwhub/items_test.go | 9 +- pkg/cwhub/sync.go | 24 ++-- pkg/hubtest/hubtest_item.go | 6 +- pkg/setup/install.go | 36 ++++-- test/bats/07_setup.bats | 25 ++-- 14 files changed, 269 insertions(+), 286 deletions(-) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index d33b2e61a3c..56f62828175 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -45,7 +45,12 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - err := hub.InstallItem(toinstall, itype, false, false) + item := hub.GetItem(itype, toinstall) + if item == nil { + log.Errorf("Item %s/%s not found in hub", itype, toinstall) + continue + } + err := item.Install(false, false) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index f934bd8561a..3fc4579cbdc 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -145,7 +145,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { updated := 0 log.Infof("Upgrading %s", itemType) for _, item := range items { - didUpdate, err := hub.UpgradeItem(itemType, item.Name, force) + didUpdate, err := item.Upgrade(force) if err != nil { return err } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 5a31dda2e36..b6beac1f4b0 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -203,7 +203,8 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) } for _, name := range args { - if hub.GetItem(it.name, name) == nil { + item := hub.GetItem(it.name, name) + if item == nil { msg := SuggestNearestMessage(hub, it.name, name) if !ignoreError { return fmt.Errorf(msg) @@ -213,11 +214,11 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) continue } - if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { + if err := item.Install(force, downloadOnly); err != nil { if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) + return fmt.Errorf("error while installing '%s': %w", item.Name, err) } - log.Errorf("Error while installing '%s': %s", name, err) + log.Errorf("Error while installing '%s': %s", item.Name, err) } } @@ -286,7 +287,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e removed := 0 for _, item := range items { - didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force) + didRemove, err := item.Remove(purge, force) if err != nil { return err } @@ -308,27 +309,25 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e } removed := 0 - for _, name := range args { - if !force { - item := hub.GetItem(it.name, name) - if item == nil { - // XXX: this should be in GetItem? - return fmt.Errorf("can't find '%s' in %s", name, it.name) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections) - log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular) - continue - } + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + if !force && len(item.BelongsToCollections) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + continue } - didRemove, err := hub.RemoveItem(it.name, name, purge, force) + didRemove, err := item.Remove(purge, force) if err != nil { return err } if didRemove { - log.Infof("Removed %s", name) + log.Infof("Removed %s", item.Name) removed++ } } @@ -392,7 +391,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) updated := 0 for _, item := range items { - didUpdate, err := hub.UpgradeItem(it.name, item.Name, force) + didUpdate, err := item.Upgrade(force) if err != nil { return err } @@ -400,7 +399,9 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) updated++ } } + log.Infof("Updated %d %s", updated, it.name) + if updated > 0 { log.Infof(ReloadMessage()) } @@ -413,13 +414,19 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) } updated := 0 - for _, name := range args { - didUpdate, err := hub.UpgradeItem(it.name, name, force) + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + didUpdate, err := item.Upgrade(force) if err != nil { return err } + if didUpdate { - log.Infof("Updated %s", name) + log.Infof("Updated %s", item.Name) updated++ } } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 2f137fa76c8..e0c20628f24 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -10,24 +10,24 @@ import ( log "github.com/sirupsen/logrus" ) -// enableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir +// enable creates a symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -func (h *Hub) enableItem(target *Item) error { - parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/") +func (i *Item) enable() error { + parentDir := filepath.Clean(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/") // create directories if needed - if target.Installed { - if target.Tainted { - return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name) + if i.Installed { + if i.Tainted { + return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) } - if target.IsLocal() { - return fmt.Errorf("%s is local, won't enable", target.Name) + if i.IsLocal() { + return fmt.Errorf("%s is local, won't enable", i.Name) } // if it's a collection, check sub-items even if the collection file itself is up-to-date - if target.UpToDate && !target.HasSubItems() { - log.Tracef("%s is installed and up-to-date, skip.", target.Name) + if i.UpToDate && !i.HasSubItems() { + log.Tracef("%s is installed and up-to-date, skip.", i.Name) return nil } } @@ -41,30 +41,30 @@ func (h *Hub) enableItem(target *Item) error { } // install sub-items if any - for _, sub := range target.SubItems() { - val, ok := h.Items[sub.Type][sub.Name] + for _, sub := range i.SubItems() { + val, ok := i.hub.Items[sub.Type][sub.Name] if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name) } - if err := h.enableItem(&val); err != nil { + if err := val.enable(); err != nil { return fmt.Errorf("while installing %s: %w", sub.Name, err) } } // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err := os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) { - log.Infof("%s already exists.", parentDir+"/"+target.FileName) + if _, err := os.Lstat(parentDir + "/" + i.FileName); !os.IsNotExist(err) { + log.Infof("%s already exists.", parentDir+"/"+i.FileName) return nil } // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) + srcPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) if err != nil { return fmt.Errorf("while getting source path: %w", err) } - dstPath, err := filepath.Abs(parentDir + "/" + target.FileName) + dstPath, err := filepath.Abs(parentDir + "/" + i.FileName) if err != nil { return fmt.Errorf("while getting destination path: %w", err) } @@ -73,37 +73,36 @@ func (h *Hub) enableItem(target *Item) error { return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) } - log.Infof("Enabled %s: %s", target.Type, target.Name) - target.Installed = true - h.Items[target.Type][target.Name] = *target + log.Infof("Enabled %s: %s", i.Type, i.Name) + i.Installed = true return nil } -func (h *Hub) purgeItem(target Item) (Item, error) { - itempath := h.local.HubDir + "/" + target.RemotePath +// purge removes the actual config file that was downloaded +func (i *Item) purge() error { + itempath := i.hub.local.HubDir + "/" + i.RemotePath // disable hub file if err := os.Remove(itempath); err != nil { - return target, fmt.Errorf("while removing file: %w", err) + return fmt.Errorf("while removing file: %w", err) } - target.Downloaded = false - log.Infof("Removed source file [%s]: %s", target.Name, itempath) - h.Items[target.Type][target.Name] = target + i.Downloaded = false + log.Infof("Removed source file [%s]: %s", i.Name, itempath) - return target, nil + return nil } -// disableItem to disable an item managed by the hub, removes the symlink if purge is true -func (h *Hub) disableItem(target *Item, purge bool, force bool) error { +// disable removes the symlink to the downloaded content, also removes the content if purge is true +func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not var err error // already disabled, noop unless purge - if !target.Installed { + if !i.Installed { if purge { - *target, err = h.purgeItem(*target) + err = i.purge() if err != nil { return err } @@ -112,20 +111,20 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { return nil } - if target.IsLocal() { - return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name) + if i.IsLocal() { + return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) } - if target.Tainted && !force { - return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name) + if i.Tainted && !force { + return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name) } // disable sub-items if any - it's a collection - for _, sub := range target.SubItems() { + for _, sub := range i.SubItems() { // XXX: we do this already when syncing, do we really need to do consistency checks here and there? - val, ok := h.Items[sub.Type][sub.Name] + val, ok := i.hub.Items[sub.Type][sub.Name] if !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name) + log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, i.Name) continue } @@ -133,14 +132,14 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { toRemove := true for _, collection := range val.BelongsToCollections { - if collection != target.Name { + if collection != i.Name { toRemove = false break } } if toRemove { - if err = h.disableItem(&val, purge, force); err != nil { + if err = val.disable(purge, force); err != nil { return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } else { @@ -148,7 +147,7 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { } } - syml, err := filepath.Abs(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) + syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName) if err != nil { return err } @@ -157,13 +156,13 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { if os.IsNotExist(err) { // we only accept to "delete" non existing items if it's a forced purge if !purge && !force { - return fmt.Errorf("can't delete %s: %s doesn't exist", target.Name, syml) + return fmt.Errorf("can't delete %s: %s doesn't exist", i.Name, syml) } } else { // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... if stat.Mode()&os.ModeSymlink == 0 { - log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml) - return fmt.Errorf("%s isn't managed by hub", target.Name) + log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) + return fmt.Errorf("%s isn't managed by hub", i.Name) } hubpath, err := os.Readlink(syml) @@ -171,14 +170,14 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { return fmt.Errorf("while reading symlink: %w", err) } - absPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath) + absPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) if err != nil { return fmt.Errorf("while abs path: %w", err) } if hubpath != absPath { - log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath) - return fmt.Errorf("%s isn't managed by hub", target.Name) + log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, absPath) + return fmt.Errorf("%s isn't managed by hub", i.Name) } // remove the symlink @@ -186,19 +185,17 @@ func (h *Hub) disableItem(target *Item, purge bool, force bool) error { return fmt.Errorf("while removing symlink: %w", err) } - log.Infof("Removed symlink [%s]: %s", target.Name, syml) + log.Infof("Removed symlink [%s]: %s", i.Name, syml) } - target.Installed = false + i.Installed = false if purge { - *target, err = h.purgeItem(*target) + err = i.purge() if err != nil { return err } } - h.Items[target.Type][target.Name] = *target - return nil } diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index a476c76d758..771edd8a7bb 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/require" ) -func testInstall(hub *Hub, t *testing.T, item Item) { +func testInstall(hub *Hub, t *testing.T, item *Item) { // Install the parser - err := hub.downloadLatest(&item, false, false) + err := item.downloadLatest(false, false) require.NoError(t, err, "failed to download %s", item.Name) err = hub.localSync() @@ -20,7 +20,7 @@ func testInstall(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - err = hub.enableItem(&item) + err = item.enable() require.NoError(t, err, "failed to enable %s", item.Name) err = hub.localSync() @@ -29,7 +29,7 @@ func testInstall(hub *Hub, t *testing.T, item Item) { assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } -func testTaint(hub *Hub, t *testing.T, item Item) { +func testTaint(hub *Hub, t *testing.T, item *Item) { assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) @@ -47,11 +47,11 @@ func testTaint(hub *Hub, t *testing.T, item Item) { assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } -func testUpdate(hub *Hub, t *testing.T, item Item) { +func testUpdate(hub *Hub, t *testing.T, item *Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := hub.downloadLatest(&item, true, true) + err := item.downloadLatest(true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status @@ -62,11 +62,11 @@ func testUpdate(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } -func testDisable(hub *Hub, t *testing.T, item Item) { +func testDisable(hub *Hub, t *testing.T, item *Item) { assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove - err := hub.disableItem(&item, false, false) + err := item.disable(false, false) require.NoError(t, err, "failed to disable %s", item.Name) // Local sync and check status @@ -79,7 +79,7 @@ func testDisable(hub *Hub, t *testing.T, item Item) { assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge - err = hub.disableItem(&item, true, false) + err = item.disable(true, false) require.NoError(t, err, "failed to purge %s", item.Name) // Local sync and check status diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index e8b49b81465..774a7386af7 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -19,15 +19,10 @@ import ( log "github.com/sirupsen/logrus" ) -// InstallItem installs an item from the hub -func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { - item := h.GetItem(itemType, name) - if item == nil { - return fmt.Errorf("unable to retrieve item: %s", name) - } - - if downloadOnly && item.Downloaded && item.UpToDate { - log.Warningf("%s is already downloaded and up-to-date", item.Name) +// Install installs the item from the hub, downloading it if needed +func (i *Item) Install(force bool, downloadOnly bool) error { + if downloadOnly && i.Downloaded && i.UpToDate { + log.Warningf("%s is already downloaded and up-to-date", i.Name) if !force { return nil @@ -35,90 +30,68 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } // XXX: confusing semantic between force and updateOnly? - if err := h.downloadLatest(item, force, true); err != nil { - return fmt.Errorf("while downloading %s: %w", item.Name, err) - } - - if err := h.AddItem(*item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) + if err := i.downloadLatest(force, true); err != nil { + return fmt.Errorf("while downloading %s: %w", i.Name, err) } if downloadOnly { // XXX: should get the path from downloadLatest - log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath)) + log.Infof("Downloaded %s to %s", i.Name, filepath.Join(i.hub.local.HubDir, i.RemotePath)) return nil } // XXX: should we stop here if the item is already installed? - if err := h.enableItem(item); err != nil { - return fmt.Errorf("while enabling %s: %w", item.Name, err) + if err := i.enable(); err != nil { + return fmt.Errorf("while enabling %s: %w", i.Name, err) } - if err := h.AddItem(*item); err != nil { - return fmt.Errorf("while adding %s: %w", item.Name, err) - } - - log.Infof("Enabled %s", item.Name) + log.Infof("Enabled %s", i.Name) return nil } -// RemoveItem disables one item, optionally removing the downloaded content -func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction bool) (bool, error) { +// Remove disables the item, optionally removing the downloaded content +func (i *Item) Remove(purge bool, forceAction bool) (bool, error) { removed := false - item := h.GetItem(itemType, name) - if item == nil { - return false, fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - if !item.Downloaded { - log.Infof("removing %s: not downloaded -- no removal required", item.Name) + if !i.Downloaded { + log.Infof("removing %s: not downloaded -- no removal required", i.Name) return false, nil } - if !item.Installed && !purge { - log.Infof("removing %s: already uninstalled", item.Name) + if !i.Installed && !purge { + log.Infof("removing %s: already uninstalled", i.Name) return false, nil } - if err := h.disableItem(item, purge, forceAction); err != nil { - return false, fmt.Errorf("unable to disable %s: %w", item.Name, err) + if err := i.disable(purge, forceAction); err != nil { + return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) } - // XXX: should take the value from disableItem + // XXX: should take the value from disable() removed = true - if err := h.AddItem(*item); err != nil { - return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err) - } - return removed, nil } -// UpgradeItem upgrades an item from the hub -func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error) { +// Upgrade downloads and applies the last version from the hub +func (i *Item) Upgrade(force bool) (bool, error) { updated := false - item := h.GetItem(itemType, name) - if item == nil { - return false, fmt.Errorf("can't find '%s' in %s", name, itemType) - } - - if !item.Downloaded { - return false, fmt.Errorf("can't upgrade %s: not installed", item.Name) + if !i.Downloaded { + return false, fmt.Errorf("can't upgrade %s: not installed", i.Name) } - if !item.Installed { - return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", item.Name) + if !i.Installed { + return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name) } - if item.UpToDate { - log.Infof("%s: up-to-date", item.Name) + if i.UpToDate { + log.Infof("%s: up-to-date", i.Name) - if err := h.DownloadDataIfNeeded(*item, force); err != nil { - return false, fmt.Errorf("%s: download failed: %w", item.Name, err) + if err := i.DownloadDataIfNeeded(force); err != nil { + return false, fmt.Errorf("%s: download failed: %w", i.Name, err) } if !force { @@ -127,112 +100,106 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error } } - if err := h.downloadLatest(item, force, true); err != nil { - return false, fmt.Errorf("%s: download failed: %w", item.Name, err) + if err := i.downloadLatest(force, true); err != nil { + return false, fmt.Errorf("%s: download failed: %w", i.Name, err) } - if !item.UpToDate { - if item.Tainted { - log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, item.Name) - } else if item.IsLocal() { - log.Infof("%v %s is local", emoji.Prohibited, item.Name) + if !i.UpToDate { + if i.Tainted { + log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) + } else if i.IsLocal() { + log.Infof("%v %s is local", emoji.Prohibited, i.Name) } } else { // a check on stdout is used while scripting to know if the hub has been upgraded // and a configuration reload is required // TODO: use a better way to communicate this - fmt.Printf("updated %s\n", item.Name) - log.Infof("%v %s: updated", emoji.Package, item.Name) + fmt.Printf("updated %s\n", i.Name) + log.Infof("%v %s: updated", emoji.Package, i.Name) updated = true } - if err := h.AddItem(*item); err != nil { - return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err) - } - return updated, nil } // downloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) downloadLatest(target *Item, overwrite bool, updateOnly bool) error { - // XXX: should return the path of the downloaded file (taken from downloadItem) - log.Debugf("Downloading %s %s", target.Type, target.Name) +func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { + // XXX: should return the path of the downloaded file (taken from download()) + log.Debugf("Downloading %s %s", i.Type, i.Name) - if !target.HasSubItems() { - if !target.Installed && updateOnly && target.Downloaded { - log.Debugf("skipping upgrade of %s: not installed", target.Name) + if !i.HasSubItems() { + if !i.Installed && updateOnly && i.Downloaded { + log.Debugf("skipping upgrade of %s: not installed", i.Name) return nil } // XXX: - return h.downloadItem(target, overwrite) + return i.download(overwrite) } // collection - for _, sub := range target.SubItems() { - val, ok := h.Items[sub.Type][sub.Name] + for _, sub := range i.SubItems() { + val, ok := i.hub.Items[sub.Type][sub.Name] if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name) + return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name) } if !val.Installed && updateOnly && val.Downloaded { - log.Debugf("skipping upgrade of %s: not installed", target.Name) + log.Debugf("skipping upgrade of %s: not installed", i.Name) continue } - log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly) + log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly) // recurse as it's a collection if sub.HasSubItems() { log.Tracef("collection, recurse") - if err := h.downloadLatest(&val, overwrite, updateOnly); err != nil { + if err := val.downloadLatest(overwrite, updateOnly); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } } downloaded := val.Downloaded - if err := h.downloadItem(&val, overwrite); err != nil { + if err := val.download(overwrite); err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { - if err := h.enableItem(&val); err != nil { + if err := val.enable(); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } - - h.Items[sub.Type][sub.Name] = val } - if err := h.downloadItem(target, overwrite); err != nil { + if err := i.download(overwrite); err != nil { return fmt.Errorf("failed to download item: %w", err) } return nil } -func (h *Hub) downloadItem(target *Item, overwrite bool) error { - url, err := h.remote.urlTo(target.RemotePath) +func (i *Item) download(overwrite bool) error { + url, err := i.hub.remote.urlTo(i.RemotePath) if err != nil { return fmt.Errorf("failed to build hub item request: %w", err) } - tdir := h.local.HubDir + tdir := i.hub.local.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { - if target.Tainted { - log.Debugf("%s: tainted, not updated", target.Name) + if i.Tainted { + log.Debugf("%s: tainted, not updated", i.Name) return nil } - if target.UpToDate { + if i.UpToDate { // We still have to check if data files are present - log.Debugf("%s: up-to-date, not updated", target.Name) + log.Debugf("%s: up-to-date, not updated", i.Name) } } @@ -253,30 +220,30 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error { hash := sha256.New() if _, err = hash.Write(body); err != nil { - return fmt.Errorf("while hashing %s: %w", target.Name, err) + return fmt.Errorf("while hashing %s: %w", i.Name, err) } meow := hex.EncodeToString(hash.Sum(nil)) - if meow != target.Versions[target.Version].Digest { + if meow != i.Versions[i.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") - log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) + log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest) - return fmt.Errorf("invalid download hash for %s", target.Name) + return fmt.Errorf("invalid download hash for %s", i.Name) } //all good, install //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/") + tmpdirs := strings.Split(tdir+"/"+i.RemotePath, "/") parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") // ensure that target file is within target dir - finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath) + finalPath, err := filepath.Abs(tdir + "/" + i.RemotePath) if err != nil { - return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err) + return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+i.RemotePath, err) } if !strings.HasPrefix(finalPath, tdir) { - return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir) + return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir) } // check dir @@ -290,13 +257,13 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error { // check actual file if _, err = os.Stat(finalPath); !os.IsNotExist(err) { - log.Warningf("%s: overwrite", target.Name) - log.Debugf("target: %s/%s", tdir, target.RemotePath) + log.Warningf("%s: overwrite", i.Name) + log.Debugf("target: %s/%s", tdir, i.RemotePath) } else { - log.Infof("%s: OK", target.Name) + log.Infof("%s: OK", i.Name) } - f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + f, err := os.OpenFile(tdir+"/"+i.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return fmt.Errorf("while opening file: %w", err) } @@ -308,22 +275,20 @@ func (h *Hub) downloadItem(target *Item, overwrite bool) error { return fmt.Errorf("while writing file: %w", err) } - target.Downloaded = true - target.Tainted = false - target.UpToDate = true + i.Downloaded = true + i.Tainted = false + i.UpToDate = true - if err = downloadData(h.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { - return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) + if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + return fmt.Errorf("while downloading data for %s: %w", i.FileName, err) } - h.Items[target.Type][target.Name] = *target - return nil } -// DownloadDataIfNeeded downloads the data files for an item -func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.local.InstallDir, target.Type, target.Stage, target.FileName) +// DownloadDataIfNeeded downloads the data files for the item +func (i *Item) DownloadDataIfNeeded(force bool) error { + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", i.hub.local.InstallDir, i.Type, i.Stage, i.FileName) itemFile, err := os.Open(itemFilePath) if err != nil { @@ -332,7 +297,7 @@ func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { defer itemFile.Close() - if err = downloadData(h.local.InstallDataDir, force, itemFile); err != nil { + if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 15f31cdca41..030dacc23fc 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -17,7 +17,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -25,8 +26,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -49,7 +49,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) require.NoError(t, err) require.True(t, didUpdate) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -68,7 +69,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -77,7 +79,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) + item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") + didRemove, err := item.Remove(false, false) require.NoError(t, err) require.True(t, didRemove) @@ -98,7 +101,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) - didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) require.NoError(t, err) require.False(t, didUpdate) @@ -125,7 +129,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + require.NoError(t, item.Install(false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -134,7 +139,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false) + item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") + didRemove, err := item.Remove(false, false) require.NoError(t, err) require.True(t, didRemove) @@ -164,7 +170,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) hub = getHubOrFail(t, hub.local, remote) - didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false) + item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") + didUpdate, err := item.Upgrade(false) require.NoError(t, err) require.True(t, didUpdate) @@ -180,7 +187,7 @@ func assertCollectionDepsInstalled(t *testing.T, collection string) { require.NoError(t, err) c := hub.Items[COLLECTIONS][collection] - require.NoError(t, hub.checkSubItems(&c)) + require.NoError(t, hub.checkSubItems(c)) } func pushUpdateToCollectionInHub() { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index f2b6c4b880b..2d4d7bc8544 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -82,6 +82,7 @@ func (h *Hub) parseIndex() error { log.Tracef("%s: %d items", itemType, len(h.Items[itemType])) for name, item := range h.Items[itemType] { + item.hub = h item.Name = name // if the item has no (redundant) author, take it from the json key @@ -92,7 +93,6 @@ func (h *Hub) parseIndex() error { item.Type = itemType x := strings.Split(item.RemotePath, "/") item.FileName = x[len(x)-1] - h.Items[itemType][name] = item // if it's a collection, check its sub-items are present // XXX should be done later, maybe report all missing at once? diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 53ccd541448..bbd2864e0da 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -26,7 +26,7 @@ const ( // The order is important, as it is used to range over sub-items in collections var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} -type HubItems map[string]map[string]Item +type HubItems map[string]map[string]*Item // ItemVersion is used to detect the version of a given item // by comparing the hash of each version to the local file. @@ -38,6 +38,9 @@ type ItemVersion struct { // Item represents an object managed in the hub. It can be a parser, scenario, collection.. type Item struct { + // back pointer to the hub, to retrieve subitems and call install/remove methods + hub *Hub + // descriptive info Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... @@ -220,23 +223,13 @@ func (i *Item) validPath(dirName, fileName string) bool { } // GetItemMap returns the map of items for a given type -func (h *Hub) GetItemMap(itemType string) map[string]Item { - m, ok := h.Items[itemType] - if !ok { - return nil - } - - return m +func (h *Hub) GetItemMap(itemType string) map[string]*Item { + return h.Items[itemType] } // GetItem returns the item from hub based on its type and full name (author/name) func (h *Hub) GetItem(itemType string, itemName string) *Item { - m, ok := h.GetItemMap(itemType)[itemName] - if !ok { - return nil - } - - return &m + return h.GetItemMap(itemType)[itemName] } // GetItemNames returns the list of item (full) names for a given type @@ -256,27 +249,14 @@ func (h *Hub) GetItemNames(itemType string) []string { return names } -// AddItem adds an item to the hub index -func (h *Hub) AddItem(item Item) error { - for _, t := range ItemTypes { - if t == item.Type { - h.Items[t][item.Name] = item - return nil - } - } - - // XXX: can this happen? - return fmt.Errorf("ItemType %s is unknown", item.Type) -} - // GetInstalledItems returns the list of installed items -func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { +func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { items, ok := h.Items[itemType] if !ok { return nil, fmt.Errorf("no %s in the hub index", itemType) } - retItems := make([]Item, 0) + retItems := make([]*Item, 0) for _, item := range items { if item.Installed { @@ -296,8 +276,8 @@ func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { retStr := make([]string, len(items)) - for i, it := range items { - retStr[i] = it.Name + for idx, it := range items { + retStr[idx] = it.Name } return retStr, nil diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index 430d6a5746d..ed11d80ae8d 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -4,8 +4,6 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestItemStatus(t *testing.T) { @@ -62,14 +60,9 @@ func TestGetters(t *testing.T) { // Add item and get it item.Name += "nope" - err := hub.AddItem(*item) - require.NoError(t, err) + hub.Items[item.Type][item.Name] = item newitem := hub.GetItem(COLLECTIONS, item.Name) require.NotNil(t, newitem) - - item.Type = "ratata" - err = hub.AddItem(*item) - cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") } } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index e9ad1b41c75..e9855027f12 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -129,20 +129,20 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { // sortedVersions returns the input data, sorted in reverse order by semver func sortedVersions(raw []string) ([]string, error) { vs := make([]*semver.Version, len(raw)) - for i, r := range raw { + for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { return nil, fmt.Errorf("%s: %w", r, err) } - vs[i] = v + vs[idx] = v } sort.Sort(sort.Reverse(semver.Collection(vs))) ret := make([]string, len(vs)) - for i, v := range vs { - ret[i] = v.Original() + for idx, v := range vs { + ret[idx] = v.Original() } return ret, nil @@ -209,7 +209,8 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { _, fileName := filepath.Split(path) - h.Items[info.ftype][info.fname] = Item{ + h.Items[info.ftype][info.fname] = &Item{ + hub: h, Name: info.fname, Stage: info.stage, Installed: true, @@ -360,7 +361,11 @@ func (h *Hub) checkSubItems(v *Item) error { continue } - if err := h.checkSubItems(&subItem); err != nil { + if err := h.checkSubItems(subItem); err != nil { + if subItem.Tainted { + v.Tainted = true + } + return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } @@ -383,8 +388,6 @@ func (h *Hub) checkSubItems(v *Item) error { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } - h.Items[sub.Type][sub.Name] = subItem - log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) } @@ -412,7 +415,7 @@ func (h *Hub) syncDir(dir string) ([]string, error) { } } - for name, item := range h.Items[COLLECTIONS] { + for _, item := range h.Items[COLLECTIONS] { if !item.Installed { continue } @@ -420,9 +423,8 @@ func (h *Hub) syncDir(dir string) ([]string, error) { vs := item.versionStatus() switch vs { case VersionUpToDate: // latest - if err := h.checkSubItems(&item); err != nil { + if err := h.checkSubItems(item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - h.Items[COLLECTIONS][name] = item } case VersionUpdateAvailable: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 329e74a5ddc..82107c6fe3a 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -410,7 +410,7 @@ func (t *HubTestItem) InstallHub() error { ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { if item.Installed { - if err := hub.DownloadDataIfNeeded(item, true); err != nil { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } @@ -422,7 +422,7 @@ func (t *HubTestItem) InstallHub() error { ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { if item.Installed { - if err := hub.DownloadDataIfNeeded(item, true); err != nil { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } @@ -434,7 +434,7 @@ func (t *HubTestItem) InstallHub() error { ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { if item.Installed { - if err := hub.DownloadDataIfNeeded(item, true); err != nil { + if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 50d16b52318..fc922c5d19b 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -62,14 +62,19 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { if len(install.Collections) > 0 { for _, collection := range setupItem.Install.Collections { + item := hub.GetItem(cwhub.COLLECTIONS, collection) + if item == nil { + return fmt.Errorf("collection %s not found", collection) + } + if dryRun { fmt.Println("dry-run: would install collection", collection) continue } - if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing collection %s: %w", collection, err) + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing collection %s: %w", item.Name, err) } } } @@ -82,8 +87,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { continue } - if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing parser %s: %w", parser, err) + item := hub.GetItem(cwhub.PARSERS, parser) + if item == nil { + return fmt.Errorf("parser %s not found", parser) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing parser %s: %w", item.Name, err) } } } @@ -96,8 +106,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { continue } - if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing scenario %s: %w", scenario, err) + item := hub.GetItem(cwhub.SCENARIOS, scenario) + if item == nil { + return fmt.Errorf("scenario %s not found", scenario) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing scenario %s: %w", item.Name, err) } } } @@ -110,8 +125,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error { continue } - if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { - return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) + item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow) + if item == nil { + return fmt.Errorf("postoverflow %s not found", postoverflow) + } + + if err := item.Install(forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err) } } } diff --git a/test/bats/07_setup.bats b/test/bats/07_setup.bats index 9d6b32d151e..1748d804072 100644 --- a/test/bats/07_setup.bats +++ b/test/bats/07_setup.bats @@ -519,6 +519,11 @@ update-notifier-motd.timer enabled enabled rune -0 cscli collections list -o json rune -0 jq -r '.collections[].name' <(output) refute_line "crowdsecurity/apache2" + + # same with dependencies + rune -0 cscli collections remove --all + rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/linux' } @test "cscli setup install-hub (dry run: install multiple collections)" { @@ -538,15 +543,17 @@ update-notifier-motd.timer enabled enabled } @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { - rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}' - assert_line 'dry-run: would install collection crowdsecurity/foo' - assert_line 'dry-run: would install collection johndoe/bar' - assert_line 'dry-run: would install parser crowdsecurity/fooparser' - assert_line 'dry-run: would install parser johndoe/barparser' - assert_line 'dry-run: would install scenario crowdsecurity/fooscenario' - assert_line 'dry-run: would install scenario johndoe/barscenario' - assert_line 'dry-run: would install postoverflow crowdsecurity/foopo' - assert_line 'dry-run: would install postoverflow johndoe/barpo' + rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-fs"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}' + assert_line 'dry-run: would install collection crowdsecurity/aws-console' + assert_line 'dry-run: would install collection crowdsecurity/caddy' + assert_line 'dry-run: would install parser crowdsecurity/asterisk-logs' + assert_line 'dry-run: would install scenario crowdsecurity/smb-fs' + assert_line 'dry-run: would install postoverflow crowdsecurity/cdn-whitelist' + assert_line 'dry-run: would install postoverflow crowdsecurity/rdns' + + rune -1 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}' + assert_stderr --partial 'collection crowdsecurity/foo not found' + } @test "cscli setup datasources" { From 9d7ed129508b9f2e013b5e6ae88a2f8763ec521c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:25:29 +0100 Subject: [PATCH 164/263] Refact cwhub (#2586) * Inspect item: always show tainted, installed, etc. when false * cleanup, comments, unused stuff * download collection content after downloading dependencies, avoid duplicate call * Return instances from Item.SubItems() * shorter i/o code * inline / simplify getData() * Handle timeout connections when downloading from hub or data --- cmd/crowdsec-cli/hubtest.go | 2 +- cmd/crowdsec-cli/item_metrics.go | 21 ++----- cmd/crowdsec-cli/itemcommands.go | 5 ++ cmd/crowdsec-cli/items.go | 2 +- cmd/crowdsec-cli/require/branch.go | 19 +++--- pkg/csconfig/cscli.go | 2 - pkg/cwhub/cwhub.go | 9 +++ pkg/cwhub/cwhub_test.go | 6 +- pkg/cwhub/dataset.go | 40 +++--------- pkg/cwhub/enable.go | 33 +++------- pkg/cwhub/enable_test.go | 11 ++-- pkg/cwhub/helpers.go | 46 +++++--------- pkg/cwhub/hub.go | 8 +-- pkg/cwhub/items.go | 98 +++++++++++++++++++---------- pkg/cwhub/remote.go | 6 +- pkg/cwhub/sync.go | 26 ++++---- test/bats/20_hub.bats | 4 +- test/bats/20_hub_collections.bats | 4 +- test/bats/20_hub_items.bats | 2 +- test/bats/20_hub_parsers.bats | 5 +- test/bats/20_hub_postoverflows.bats | 4 +- test/bats/20_hub_scenarios.bats | 5 +- 22 files changed, 162 insertions(+), 196 deletions(-) diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 5052c133292..8b574c3ee86 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -136,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios } configFilePath := filepath.Join(testPath, "config.yaml") - fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + fd, err := os.Create(configFilePath) if err != nil { return fmt.Errorf("open: %s", err) } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index 4442a332e52..51b652abc27 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -18,8 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -// XXX: this should not need hub? -func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) error { +func ShowMetrics(hubItem *cwhub.Item) error { switch hubItem.Type { case cwhub.PARSERS: metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) @@ -28,25 +27,13 @@ func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) error { metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) scenarioMetricsTable(color.Output, hubItem.Name, metrics) case cwhub.COLLECTIONS: - for _, parserName := range hubItem.Parsers { - metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, parserName) - parserMetricsTable(color.Output, parserName, metrics) - } - for _, scenarioName := range hubItem.Scenarios { - metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, scenarioName) - scenarioMetricsTable(color.Output, scenarioName, metrics) - } - for _, collName := range hubItem.Collections { - subColl := hub.GetItem(cwhub.COLLECTIONS, collName) - if subColl == nil { - return fmt.Errorf("unable to retrieve sub-collection '%s' from '%s'", collName, hubItem.Name) - } - if err := ShowMetrics(hub, subColl); err != nil { + for _, sub := range hubItem.SubItems() { + if err := ShowMetrics(sub); err != nil { return err } } default: - log.Errorf("item of type '%s' is unknown", hubItem.Type) + // no metrics for this item type } return nil } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index b6beac1f4b0..3880afc146d 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -211,6 +211,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) } log.Errorf(msg) + continue } @@ -309,6 +310,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e } removed := 0 + for _, itemName := range args { item := hub.GetItem(it.name, itemName) if item == nil { @@ -318,6 +320,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e if !force && len(item.BelongsToCollections) > 0 { log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections) log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + continue } @@ -390,6 +393,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) } updated := 0 + for _, item := range items { didUpdate, err := item.Upgrade(force) if err != nil { @@ -414,6 +418,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) } updated := 0 + for _, itemName := range args { item := hub.GetItem(it.name, itemName) if item == nil { diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 6d913618363..6d84582dbed 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -157,7 +157,7 @@ func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error { if csConfig.Cscli.Output == "human" && showMetrics { fmt.Printf("\nCurrent metrics: \n") - if err := ShowMetrics(hub, item); err != nil { + if err := ShowMetrics(item); err != nil { return err } } diff --git a/cmd/crowdsec-cli/require/branch.go b/cmd/crowdsec-cli/require/branch.go index 14d48279652..b82f34a2722 100644 --- a/cmd/crowdsec-cli/require/branch.go +++ b/cmd/crowdsec-cli/require/branch.go @@ -10,32 +10,33 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string { +func chooseBranch(cfg *csconfig.Config) string { + // this was set from config.yaml or flag if cfg.Cscli.HubBranch != "" { - logger.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch) + log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch) return cfg.Cscli.HubBranch } latest, err := cwversion.Latest() if err != nil { - logger.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err) + log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err) return "master" } csVersion := cwversion.VersionStrip() if csVersion == latest { - logger.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion) + log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion) return "master" } // if current version is greater than the latest we are in pre-release if semver.Compare(csVersion, latest) == 1 { - logger.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion) + log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion) return "master" } if csVersion == "" { - logger.Warning("Crowdsec version is not set, using hub branch 'master'") + log.Warning("Crowdsec version is not set, using hub branch 'master'") return "master" } @@ -49,11 +50,7 @@ func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string { // HubBranch sets the branch (in cscli config) and returns its value // It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag func HubBranch(cfg *csconfig.Config) string { - // XXX: we want to be able to suppress logs - // to avoid being too noisy in some commands - logger := log.StandardLogger() - - branch := chooseBranch(cfg, logger) + branch := chooseBranch(cfg) cfg.Cscli.HubBranch = branch diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 6ecce4a53f2..2a3fa7df3b2 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -21,8 +21,6 @@ func (c *Config) loadCSCLI() error { c.Cscli = &CscliCfg{} } - // XXX: HubBranch default should be set here and fed to HubCfg? - if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 8a68c97a5f9..fa59d63da34 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -3,3 +3,12 @@ // This includes retrieving the index, the items to install (parsers, scenarios, data files...) // and managing the dependencies and taints. package cwhub + +import ( + "net/http" + "time" +) + +var hubClient = &http.Client{ + Timeout: 10 * time.Second, +} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 3e1bb57d151..af19b93ee7c 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -79,14 +79,14 @@ func envSetup(t *testing.T) *Hub { setResponseByPath() log.SetLevel(log.DebugLevel) - defaultTransport := http.DefaultClient.Transport + defaultTransport := hubClient.Transport t.Cleanup(func() { - http.DefaultClient.Transport = defaultTransport + hubClient.Transport = defaultTransport }) // Mock the http client - http.DefaultClient.Transport = newMockTransport() + hubClient.Transport = newMockTransport() hub := testHub(t, true) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 2b3629a978d..a0e710ebd1e 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -21,7 +21,7 @@ type DataSet struct { func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) - resp, err := http.DefaultClient.Get(url) + resp, err := hubClient.Get(url) if err != nil { return fmt.Errorf("while downloading %s: %w", url, err) } @@ -31,18 +31,13 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("while downloading %s: %w", url, err) - } - - file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) + file, err := os.Create(destPath) if err != nil { return err } defer file.Close() - _, err = file.Write(body) + _, err = io.Copy(file, resp.Body) if err != nil { return err } @@ -54,19 +49,6 @@ func downloadFile(url string, destPath string) error { return nil } -func getData(data []types.DataSource, dataDir string) error { - for _, dataS := range data { - destPath := filepath.Join(dataDir, dataS.DestPath) - log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) - - if err := downloadFile(dataS.SourceURL, destPath); err != nil { - return err - } - } - - return nil -} - // downloadData downloads the data files for an item func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) @@ -82,17 +64,15 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { return fmt.Errorf("while reading file: %w", err) } - download := false - for _, dataS := range data.Data { - if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) { - download = true - } - } + destPath := filepath.Join(dataFolder, dataS.DestPath) + + if _, err := os.Stat(destPath); os.IsNotExist(err) || force { + log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) - if download || force { - if err := getData(data.Data, dataFolder); err != nil { - return fmt.Errorf("while getting data: %w", err) + if err := downloadFile(dataS.SourceURL, destPath); err != nil { + return fmt.Errorf("while getting data: %w", err) + } } } } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index e0c20628f24..4886899fdc9 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -42,12 +42,7 @@ func (i *Item) enable() error { // install sub-items if any for _, sub := range i.SubItems() { - val, ok := i.hub.Items[sub.Type][sub.Name] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name) - } - - if err := val.enable(); err != nil { + if err := sub.enable(); err != nil { return fmt.Errorf("while installing %s: %w", sub.Name, err) } } @@ -102,8 +97,7 @@ func (i *Item) disable(purge bool, force bool) error { // already disabled, noop unless purge if !i.Installed { if purge { - err = i.purge() - if err != nil { + if err = i.purge(); err != nil { return err } } @@ -121,29 +115,22 @@ func (i *Item) disable(purge bool, force bool) error { // disable sub-items if any - it's a collection for _, sub := range i.SubItems() { - // XXX: we do this already when syncing, do we really need to do consistency checks here and there? - val, ok := i.hub.Items[sub.Type][sub.Name] - if !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, i.Name) - continue - } - // check if the item doesn't belong to another collection before removing it - toRemove := true + removeSub := true - for _, collection := range val.BelongsToCollections { + for _, collection := range sub.BelongsToCollections { if collection != i.Name { - toRemove = false + removeSub = false break } } - if toRemove { - if err = val.disable(purge, force); err != nil { + if removeSub { + if err = sub.disable(purge, force); err != nil { return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } else { - log.Infof("%s was not removed because it belongs to another collection", val.Name) + log.Infof("%s was not removed because it belongs to another collection", sub.Name) } } @@ -180,7 +167,6 @@ func (i *Item) disable(purge bool, force bool) error { return fmt.Errorf("%s isn't managed by hub", i.Name) } - // remove the symlink if err = os.Remove(syml); err != nil { return fmt.Errorf("while removing symlink: %w", err) } @@ -191,8 +177,7 @@ func (i *Item) disable(purge bool, force bool) error { i.Installed = false if purge { - err = i.purge() - if err != nil { + if err = i.purge(); err != nil { return err } } diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 771edd8a7bb..9173024a4b0 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -32,13 +32,10 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { func testTaint(hub *Hub, t *testing.T, item *Item) { assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) - require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) - - defer f.Close() - - _, err = f.WriteString("tainted") - require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) + // truncate the file + f, err := os.Create(item.LocalPath) + require.NoError(t, err) + f.Close() // Local sync and check status err = hub.localSync() diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 774a7386af7..9eeb80088bc 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -127,24 +127,8 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { // XXX: should return the path of the downloaded file (taken from download()) log.Debugf("Downloading %s %s", i.Type, i.Name) - if !i.HasSubItems() { - if !i.Installed && updateOnly && i.Downloaded { - log.Debugf("skipping upgrade of %s: not installed", i.Name) - return nil - } - - // XXX: - return i.download(overwrite) - } - - // collection for _, sub := range i.SubItems() { - val, ok := i.hub.Items[sub.Type][sub.Name] - if !ok { - return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, i.Name) - } - - if !val.Installed && updateOnly && val.Downloaded { + if !sub.Installed && updateOnly && sub.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) continue } @@ -155,26 +139,31 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { if sub.HasSubItems() { log.Tracef("collection, recurse") - if err := val.downloadLatest(overwrite, updateOnly); err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) + if err := sub.downloadLatest(overwrite, updateOnly); err != nil { + return fmt.Errorf("while downloading %s: %w", sub.Name, err) } } - downloaded := val.Downloaded + downloaded := sub.Downloaded - if err := val.download(overwrite); err != nil { - return fmt.Errorf("while downloading %s: %w", val.Name, err) + if err := sub.download(overwrite); err != nil { + return fmt.Errorf("while downloading %s: %w", sub.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. - // We check if val.Downloaded is false because maybe the item has been disabled by the user. - if !val.Installed && !downloaded { - if err := val.enable(); err != nil { - return fmt.Errorf("enabling '%s': %w", val.Name, err) + // We check if sub.Downloaded is false because maybe the item has been disabled by the user. + if !sub.Installed && !downloaded { + if err := sub.enable(); err != nil { + return fmt.Errorf("enabling '%s': %w", sub.Name, err) } } } + if !i.Installed && updateOnly && i.Downloaded { + log.Debugf("skipping upgrade of %s: not installed", i.Name) + return nil + } + if err := i.download(overwrite); err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -203,7 +192,7 @@ func (i *Item) download(overwrite bool) error { } } - resp, err := http.DefaultClient.Get(url) + resp, err := hubClient.Get(url) if err != nil { return fmt.Errorf("while downloading %s: %w", url, err) } @@ -263,11 +252,10 @@ func (i *Item) download(overwrite bool) error { log.Infof("%s: OK", i.Name) } - f, err := os.OpenFile(tdir+"/"+i.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + f, err := os.Create(tdir + "/" + i.RemotePath) if err != nil { return fmt.Errorf("while opening file: %w", err) } - defer f.Close() _, err = f.Write(body) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 2d4d7bc8544..1fc8e40ad35 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -94,13 +94,7 @@ func (h *Hub) parseIndex() error { x := strings.Split(item.RemotePath, "/") item.FileName = x[len(x)-1] - // if it's a collection, check its sub-items are present - // XXX should be done later, maybe report all missing at once? - for _, sub := range item.SubItems() { - if _, ok := h.Items[sub.Type][sub.Name]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name) - } - } + item.logMissingSubItems() } } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index bbd2864e0da..e9a1b8d4bcf 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -6,6 +6,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/enescakir/emoji" + log "github.com/sirupsen/logrus" ) const ( @@ -32,8 +33,8 @@ type HubItems map[string]map[string]*Item // by comparing the hash of each version to the local file. // If the item does not match any known version, it is considered tainted. type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` } // Item represents an object managed in the hub. It can be a parser, scenario, collection.. @@ -60,10 +61,10 @@ type Item struct { LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} LocalVersion string `json:"local_version,omitempty"` LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed,omitempty"` // XXX: should we remove omitempty from bool fields? - Downloaded bool `json:"downloaded,omitempty"` - UpToDate bool `json:"up_to_date,omitempty"` - Tainted bool `json:"tainted,omitempty"` // has it been locally modified? + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` // has it been locally modified? // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` @@ -72,19 +73,10 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } -type SubItem struct { - Type string - Name string -} - func (i *Item) HasSubItems() bool { return i.Type == COLLECTIONS } -func (i *SubItem) HasSubItems() bool { - return i.Type == COLLECTIONS -} - func (i *Item) IsLocal() bool { return i.Installed && !i.Downloaded } @@ -97,7 +89,7 @@ func (i Item) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Alias - Local bool `json:"local"` // XXX: omitempty? + Local bool `json:"local"` }{ Alias: Alias(i), Local: i.IsLocal(), @@ -119,39 +111,79 @@ func (i Item) MarshalYAML() (interface{}, error) { }, nil } -// SubItems returns the list of sub items for a given item (typically a collection) -func (i *Item) SubItems() []SubItem { - sub := make([]SubItem, - len(i.Parsers)+ - len(i.PostOverflows)+ - len(i.Scenarios)+ - len(i.Collections)) - - n := 0 +// SubItems returns a slice of sub-item pointers, excluding the ones that were not found +func (i *Item) SubItems() []*Item { + sub := make([]*Item, 0) for _, name := range i.Parsers { - sub[n] = SubItem{Type: PARSERS, Name: name} - n++ + s := i.hub.GetItem(PARSERS, name) + if s == nil { + continue + } + + sub = append(sub, s) } for _, name := range i.PostOverflows { - sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name} - n++ + s := i.hub.GetItem(POSTOVERFLOWS, name) + if s == nil { + continue + } + + sub = append(sub, s) } for _, name := range i.Scenarios { - sub[n] = SubItem{Type: SCENARIOS, Name: name} - n++ + s := i.hub.GetItem(SCENARIOS, name) + if s == nil { + continue + } + + sub = append(sub, s) } for _, name := range i.Collections { - sub[n] = SubItem{Type: COLLECTIONS, Name: name} - n++ + s := i.hub.GetItem(COLLECTIONS, name) + if s == nil { + continue + } + + sub = append(sub, s) } return sub } +func (i *Item) logMissingSubItems() { + if !i.HasSubItems() { + return + } + + for _, subName := range i.Parsers { + if i.hub.GetItem(PARSERS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) + } + } + + for _, subName := range i.Scenarios { + if i.hub.GetItem(SCENARIOS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) + } + } + + for _, subName := range i.PostOverflows { + if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) + } + } + + for _, subName := range i.Collections { + if i.hub.GetItem(COLLECTIONS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) + } + } +} + // Status returns the status of the item as a string and an emoji // ie. "enabled,update-available" and emoji.Warning func (i *Item) Status() (string, emoji.Emoji) { diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 9e0c2c04db4..ad97a4efbb5 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -22,6 +22,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return "", ErrNilRemoteHub } + // the template must contain two string placeholders if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate { return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate) } @@ -40,7 +41,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { return fmt.Errorf("failed to build hub index request: %w", err) } - resp, err := http.DefaultClient.Get(url) + resp, err := hubClient.Get(url) if err != nil { return fmt.Errorf("failed http request for hub index: %w", err) } @@ -69,8 +70,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { return nil } - file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) - + file, err := os.Create(localPath) if err != nil { return fmt.Errorf("while opening hub index file: %w", err) } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index e9855027f12..fe47342cabc 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -129,6 +129,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { // sortedVersions returns the input data, sorted in reverse order by semver func sortedVersions(raw []string) ([]string, error) { vs := make([]*semver.Version, len(raw)) + for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { @@ -180,7 +181,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { } /* - we can encounter 'collections' in the form of a symlink : + we can encounter 'collections' in the form of a symlink: /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml when the collection is installed, both files are created */ @@ -350,42 +351,37 @@ func (h *Hub) checkSubItems(v *Item) error { log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) for _, sub := range v.SubItems() { - subItem, ok := h.Items[sub.Type][sub.Name] - if !ok { - return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name) - } - - log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) + log.Tracef("check %s installed:%t", sub.Name, sub.Installed) if !v.Installed { continue } - if err := h.checkSubItems(subItem); err != nil { - if subItem.Tainted { + if err := h.checkSubItems(sub); err != nil { + if sub.Tainted { v.Tainted = true } - return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) + return fmt.Errorf("sub collection %s is broken: %w", sub.Name, err) } - if subItem.Tainted { + if sub.Tainted { v.Tainted = true return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) } - if !subItem.Installed && v.Installed { + if !sub.Installed && v.Installed { v.Tainted = true return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) } - if !subItem.UpToDate { + if !sub.UpToDate { v.UpToDate = false return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) } - if !slices.Contains(subItem.BelongsToCollections, v.Name) { - subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) + if !slices.Contains(sub.BelongsToCollections, v.Name) { + sub.BelongsToCollections = append(sub.BelongsToCollections, v.Name) } log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index f6c1c87cfee..0a45b08398c 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -66,8 +66,8 @@ teardown() { new_hub=$(jq <"$HUB_DIR/.index.json" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') echo "$new_hub" >"$HUB_DIR/.index.json" rune -0 cscli hub list --error - assert_stderr --partial "Referred parsers crowdsecurity/smb-logs in collection crowdsecurity/smb doesn't exist." - assert_stderr --partial "Referred scenarios crowdsecurity/mysql-bf in collection crowdsecurity/mysql doesn't exist." + assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" + assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" } @test "cscli hub update" { diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 8452fc66680..b527e39050a 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -185,7 +185,7 @@ teardown() { rune -0 cscli collections inspect crowdsecurity/sshd -o json rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) # XXX: .installed is missing -- not false - assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null]' + assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]' # one item, raw rune -0 cscli collections inspect crowdsecurity/sshd -o raw @@ -211,7 +211,7 @@ teardown() { # multiple items, json rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) - assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",null]]' + assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",false]]' # multiple items, raw rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 6ac19bf90b2..80e0934c6e6 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -53,7 +53,7 @@ teardown() { rune -0 cscli collections inspect crowdsecurity/sshd -o json # XXX: is this supposed to be tainted or up to date? rune -0 jq -c '[.local_version,.up_to_date,.tainted]' <(output) - assert_json '["1.10",null,null]' + assert_json '["1.10",false,false]' } @test "hub index with invalid (non semver) version numbers" { diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 05088cde96b..544f03aeff2 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -187,8 +187,7 @@ teardown() { # one item, json rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) - # XXX: .installed is missing -- not false - assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null]' + assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false]' # one item, raw rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw @@ -215,7 +214,7 @@ teardown() { # multiple items, json rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) - assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",null]]' + assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",false]]' # multiple items, raw rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index 7e3714de9d6..9f766a2ad40 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -189,7 +189,7 @@ teardown() { rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) # XXX: .installed is missing -- not false - assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null]' + assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]' # one item, raw rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw @@ -216,7 +216,7 @@ teardown() { # multiple items, json rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) - assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",null]]' + assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",false]]' # multiple items, raw rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index eb319382316..7cbbb010453 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -187,8 +187,7 @@ teardown() { # one item, json rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) - # XXX: .installed is missing -- not false - assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null]' + assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false]' # one item, raw rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw @@ -214,7 +213,7 @@ teardown() { # multiple items, json rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) - assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",null]]' + assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",false]]' # multiple items, raw rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw From d5c78708260898082e8928c1da163d23b938ef7a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:32:12 +0100 Subject: [PATCH 165/263] Refact cwhub: remove global hub instance (#2587) * csConfig.Cscli is always loaded now, configuration paths too * Remove global/singleton hub instance --- cmd/crowdsec-cli/hub.go | 25 ++----------------------- cmd/crowdsec-cli/main.go | 4 ---- cmd/crowdsec-cli/simulation.go | 5 +---- cmd/crowdsec/crowdsec.go | 15 +++++---------- cmd/crowdsec/main.go | 2 +- cmd/crowdsec/output.go | 8 ++------ cmd/crowdsec/serve.go | 19 +++++++++++++++---- pkg/cwhub/helpers_test.go | 13 +++++-------- pkg/cwhub/hub.go | 21 ++++++--------------- pkg/cwhub/hub_test.go | 3 --- pkg/leakybucket/buckets_test.go | 11 ++++++----- pkg/leakybucket/manager_load.go | 9 ++------- 12 files changed, 45 insertions(+), 90 deletions(-) diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 3fc4579cbdc..fe0e8b8cfd1 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -24,13 +24,6 @@ cscli hub update cscli hub upgrade`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - return nil - }, } cmdHub.AddCommand(NewHubListCmd()) @@ -110,14 +103,7 @@ Fetches the .index.json file from the hub, containing the list of available conf `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - return nil - }, - RunE: runHubUpdate, + RunE: runHubUpdate, } return cmdHubUpdate @@ -168,14 +154,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before interacting with hub") - } - - return nil - }, - RunE: runHubUpgrade, + RunE: runHubUpgrade, } flags := cmdHubUpgrade.Flags() diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index da75ed439db..56786af52ab 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -64,10 +64,6 @@ func initConfig() { log.Debugf("Enabled feature flags: %s", fflist) } - if csConfig.Cscli == nil { - log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) - } - if flagBranch != "" { csConfig.Cscli.HubBranch = flagBranch } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 83f01bc15a9..c43e22a68da 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -3,11 +3,11 @@ package main import ( "fmt" "os" - "slices" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "slices" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" @@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`, if err := csConfig.LoadSimulation(); err != nil { log.Fatal(err) } - if csConfig.Cscli == nil { - return fmt.Errorf("you must configure cli before using simulation") - } if csConfig.Cscli.SimulationConfig == nil { return fmt.Errorf("no simulation configured") } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 3b3a69cd33f..fc1fdb9463d 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -20,14 +20,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { +func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { var err error - hub, err := cwhub.NewHub(cConfig.Hub, nil, false) - if err != nil { - return nil, fmt.Errorf("while loading hub index: %w", err) - } - // Start loading configs csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { @@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { return csParsers, nil } -func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { +func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error { inputEventChan = make(chan types.Event) inputLineChan = make(chan types.Event) @@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ { outputsTomb.Go(func() error { defer trace.CatchPanic("crowdsec/runOutput") - if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil { + if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil { log.Fatalf("starting outputs error : %s", err) return err } @@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { return nil } -func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) { +func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) { crowdsecTomb.Go(func() error { defer trace.CatchPanic("crowdsec/serveCrowdsec") go func() { @@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady // this logs every time, even at config reload log.Debugf("running agent after %s ms", time.Since(crowdsecT0)) agentReady <- true - if err := runCrowdsec(cConfig, parsers); err != nil { + if err := runCrowdsec(cConfig, parsers, hub); err != nil { log.Fatalf("unable to start crowdsec routines: %s", err) } }() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index d4c2ceb9095..61bbfad86ff 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { buckets = leakybucket.NewBuckets() log.Infof("Loading %d scenario files", len(files)) - holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent) + holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { return fmt.Errorf("scenario loading failed: %v", err) diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 95235aa9ad7..0abb8a9c9d2 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error var bucketOverflows []types.Event func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, - postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error { + postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, + apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error { var err error ticker := time.NewTicker(1 * time.Second) @@ -70,11 +71,6 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky var cache []types.RuntimeAlert var cacheMutex sync.Mutex - hub, err := cwhub.GetHub() - if err != nil { - return err - } - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("loading list of installed hub scenarios: %w", err) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 8513e004608..d51344e6b9e 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" @@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } if !cConfig.DisableAgent { - csParsers, err := initCrowdsec(cConfig) + hub, err := cwhub.NewHub(cConfig.Hub, nil, false) + if err != nil { + return nil, fmt.Errorf("while loading hub index: %w", err) + } + + csParsers, err := initCrowdsec(cConfig, hub) if err != nil { return nil, fmt.Errorf("unable to init crowdsec: %w", err) } @@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } agentReady := make(chan bool, 1) - serveCrowdsec(csParsers, cConfig, agentReady) + serveCrowdsec(csParsers, cConfig, hub, agentReady) } log.Printf("Reload is finished") @@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if !cConfig.DisableAgent { - csParsers, err := initCrowdsec(cConfig) + hub, err := cwhub.NewHub(cConfig.Hub, nil, false) + if err != nil { + return fmt.Errorf("while loading hub index: %w", err) + } + + csParsers, err := initCrowdsec(cConfig, hub) if err != nil { return fmt.Errorf("crowdsec init: %w", err) } // if it's just linting, we're done if !flags.TestMode { - serveCrowdsec(csParsers, cConfig, agentReady) + serveCrowdsec(csParsers, cConfig, hub, agentReady) } } else { agentReady <- true diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 030dacc23fc..0ad647b9917 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -28,7 +28,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { // This is the scenario that gets added in next version of collection require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() @@ -53,7 +53,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { didUpdate, err := item.Upgrade(false) require.NoError(t, err) require.True(t, didUpdate) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) @@ -77,7 +77,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") didRemove, err := item.Remove(false, false) @@ -137,7 +137,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") + assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") didRemove, err := item.Remove(false, false) @@ -180,12 +180,9 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } -func assertCollectionDepsInstalled(t *testing.T, collection string) { +func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) { t.Helper() - hub, err := GetHub() - require.NoError(t, err) - c := hub.Items[COLLECTIONS][collection] require.NoError(t, hub.checkSubItems(c)) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 1fc8e40ad35..d27264694bf 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -20,17 +20,8 @@ type Hub struct { Warnings []string } -var theHub *Hub - -// GetHub returns the hub singleton -// it returns an error if it's not initialized to avoid nil dereference -// XXX: convenience function that we should get rid of at some point -func GetHub() (*Hub, error) { - if theHub == nil { - return nil, fmt.Errorf("hub not initialized") - } - - return theHub, nil +func (h *Hub) GetDataDir() string { + return h.local.InstallDataDir } // NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state @@ -48,20 +39,20 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo log.Debugf("loading hub idx %s", local.HubIndexFile) - theHub = &Hub{ + hub := &Hub{ local: local, remote: remote, } - if err := theHub.parseIndex(); err != nil { + if err := hub.parseIndex(); err != nil { return nil, fmt.Errorf("failed to load index: %w", err) } - if err := theHub.localSync(); err != nil { + if err := hub.localSync(); err != nil { return nil, fmt.Errorf("failed to sync items: %w", err) } - return theHub, nil + return hub, nil } // parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 21bb2d1fefd..1bba5e90a60 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -21,9 +21,6 @@ func TestInitHubUpdate(t *testing.T) { _, err := NewHub(hub.local, remote, true) require.NoError(t, err) - - _, err = GetHub() - require.NoError(t, err) } func TestDownloadIndex(t *testing.T) { diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index b47b0717ea0..f74fb835576 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -41,9 +41,10 @@ func TestBucket(t *testing.T) { hubCfg := &csconfig.LocalHubCfg{ HubDir: filepath.Join(testdata, "hub"), HubIndexFile: filepath.Join(testdata, "hub", "index.json"), + InstallDataDir: testdata, } - _, err := cwhub.NewHub(hubCfg, nil, false) + hub, err := cwhub.NewHub(hubCfg, nil, false) if err != nil { t.Fatalf("failed to init hub: %s", err) } @@ -54,7 +55,7 @@ func TestBucket(t *testing.T) { } if envSetting != "" { - if err := testOneBucket(t, envSetting, tomb); err != nil { + if err := testOneBucket(t, hub, envSetting, tomb); err != nil { t.Fatalf("Test '%s' failed : %s", envSetting, err) } } else { @@ -72,7 +73,7 @@ func TestBucket(t *testing.T) { tomb.Go(func() error { wg.Add(1) defer wg.Done() - if err := testOneBucket(t, fname, tomb); err != nil { + if err := testOneBucket(t, hub, fname, tomb); err != nil { t.Fatalf("Test '%s' failed : %s", fname, err) } return nil @@ -94,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) { } } -func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { +func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error { var ( holders []BucketFactory @@ -131,7 +132,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { } cscfg := &csconfig.CrowdsecServiceCfg{} - holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false) + holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false) if err != nil { t.Fatalf("failed loading bucket : %s", err) } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 484d6b9ac81..0e84d9547f1 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -179,17 +179,12 @@ func ValidateFactory(bucketFactory *BucketFactory) error { return nil } -func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { +func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { var ( ret = []BucketFactory{} response chan types.Event ) - hub, err := cwhub.GetHub() - if err != nil { - return nil, nil, err - } - response = make(chan types.Event, 1) for _, f := range files { log.Debugf("Loading '%s'", f) @@ -217,7 +212,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []str log.Tracef("End of yaml file") break } - bucketFactory.DataDir = dataDir + bucketFactory.DataDir = hub.GetDataDir() //check empty if bucketFactory.Name == "" { log.Errorf("Won't load nameless bucket") From 4bfa0a7b4db89cd28cdf938dcbbe5d0d16d0409f Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 10 Nov 2023 17:33:53 +0100 Subject: [PATCH 166/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 7 +- pkg/types/event.go | 226 ------------------- pkg/types/waap_event.go | 231 ++++++++++++++++++++ 3 files changed, 235 insertions(+), 229 deletions(-) create mode 100644 pkg/types/waap_event.go diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 5bf8cef636d..fa80141ed86 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -40,10 +40,11 @@ func (r *WaapRunner) Init(datadir string) error { for _, collection := range r.WaapRuntime.OutOfBandRules { outOfBandRules += collection.String() } - runnerLogger := r.logger.Dup() + inBandLogger := r.logger.Dup().WithField("band", "inband") + outBandLogger := r.logger.Dup().WithField("band", "outband") //setting up inband engine - inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(runnerLogger)) + inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(inBandLogger)) if !r.WaapRuntime.Config.InbandOptions.DisableBodyInspection { inbandCfg = inbandCfg.WithRequestBodyAccess() } else { @@ -58,7 +59,7 @@ func (r *WaapRunner) Init(datadir string) error { } //setting up outband engine - outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(runnerLogger)) + outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(outBandLogger)) if !r.WaapRuntime.Config.OutOfBandOptions.DisableBodyInspection { outbandCfg = outbandCfg.WithRequestBodyAccess() } else { diff --git a/pkg/types/event.go b/pkg/types/event.go index 83e5ee76a13..0e8de7f8e29 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -1,9 +1,7 @@ package types import ( - "fmt" "net" - "regexp" "time" log "github.com/sirupsen/logrus" @@ -18,230 +16,6 @@ const ( WAAP ) -/* - 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban - 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban - - -any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) - -len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 - -*/ - -type MatchedRules []map[string]interface{} - -type WaapEvent struct { - MatchedRules - Vars map[string]string -} -type Field string - -func (f Field) String() string { - return fmt.Sprintf("%s", f) -} - -const ( - ID Field = "id" - RuleType Field = "rule_type" - Tags Field = "tags" - File Field = "file" - Confidence Field = "confidence" - Revision Field = "revision" - SecMark Field = "secmark" - Accuracy Field = "accuracy" - Msg Field = "msg" - Severity Field = "severity" - Kind Field = "kind" -) - -func (w WaapEvent) GetVar(varName string) string { - if w.Vars == nil { - return "" - } - if val, ok := w.Vars[varName]; ok { - return val - } - log.Infof("var %s not found", varName, w.Vars) - return "" - -} - -// getters -func (w MatchedRules) GetField(field Field) []interface{} { - ret := make([]interface{}, 0) - for _, rule := range w { - ret = append(ret, rule[field.String()]) - } - return ret -} - -func (w MatchedRules) GetURI() string { - for _, rule := range w { - return rule["uri"].(string) - } - return "" -} - -func (w MatchedRules) GetHash() string { - for _, rule := range w { - //@sbl : let's fix this - return rule["hash"].(string) - } - return "" -} - -func (w MatchedRules) GetVersion() string { - for _, rule := range w { - //@sbl : let's fix this - return rule["version"].(string) - } - return "" -} - -func (w MatchedRules) GetName() string { - for _, rule := range w { - //@sbl : let's fix this - return rule["name"].(string) - } - return "" -} - -func (w MatchedRules) GetMethod() string { - for _, rule := range w { - return rule["method"].(string) - } - return "" -} - -func (w MatchedRules) GetRuleIDs() []int { - ret := make([]int, 0) - for _, rule := range w { - ret = append(ret, rule["id"].(int)) - } - return ret -} - -func (w MatchedRules) Kinds() []string { - ret := make([]string, 0) - for _, rule := range w { - exists := false - for _, val := range ret { - if val == rule["kind"] { - exists = true - break - } - } - if !exists { - ret = append(ret, rule["kind"].(string)) - } - } - return ret -} - -// filters -func (w MatchedRules) ByID(id int) MatchedRules { - waap := MatchedRules{} - - for _, rule := range w { - if rule["id"] == id { - waap = append(waap, rule) - } - } - return waap -} - -func (w MatchedRules) ByKind(kind string) MatchedRules { - waap := MatchedRules{} - for _, rule := range w { - if rule["kind"] == kind { - waap = append(waap, rule) - } - } - return waap -} - -func (w MatchedRules) ByTags(match []string) MatchedRules { - waap := MatchedRules{} - for _, rule := range w { - for _, tag := range rule["tags"].([]string) { - for _, match_tag := range match { - if tag == match_tag { - waap = append(waap, rule) - break - } - } - } - } - return waap -} - -func (w MatchedRules) ByTag(match string) MatchedRules { - waap := MatchedRules{} - for _, rule := range w { - for _, tag := range rule["tags"].([]string) { - if tag == match { - waap = append(waap, rule) - break - } - } - } - return waap -} - -func (w MatchedRules) ByTagRx(rx string) MatchedRules { - waap := MatchedRules{} - re := regexp.MustCompile(rx) - if re == nil { - return waap - } - for _, rule := range w { - for _, tag := range rule["tags"].([]string) { - log.Infof("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) - if re.MatchString(tag) { - waap = append(waap, rule) - break - } - } - } - return waap -} - -func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules { - log.Infof("%s", w) - wap := MatchedRules{} - for _, rule := range w { - if rule["disruptive"] == is { - wap = append(wap, rule) - } - } - log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap)) - - return wap -} - -func (w MatchedRules) BySeverity(severity string) MatchedRules { - wap := MatchedRules{} - for _, rule := range w { - if rule["severity"] == severity { - wap = append(wap, rule) - } - } - log.Infof("BySeverity(%s) -> %d", severity, len(wap)) - return wap -} - -func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules { - wap := MatchedRules{} - for _, rule := range w { - if rule["accuracy"] == accuracy { - wap = append(wap, rule) - } - } - log.Infof("ByAccuracy(%s) -> %d", accuracy, len(wap)) - return wap -} - // Event is the structure representing a runtime event (log or overflow) type Event struct { /* is it a log or an overflow */ diff --git a/pkg/types/waap_event.go b/pkg/types/waap_event.go new file mode 100644 index 00000000000..9b474be4ddd --- /dev/null +++ b/pkg/types/waap_event.go @@ -0,0 +1,231 @@ +package types + +import ( + "regexp" + + log "github.com/sirupsen/logrus" +) + +/* + 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban + 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban + + +any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) + +len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 + +*/ + +type MatchedRules []map[string]interface{} + +type WaapEvent struct { + MatchedRules + Vars map[string]string +} +type Field string + +func (f Field) String() string { + return string(f) +} + +const ( + ID Field = "id" + RuleType Field = "rule_type" + Tags Field = "tags" + File Field = "file" + Confidence Field = "confidence" + Revision Field = "revision" + SecMark Field = "secmark" + Accuracy Field = "accuracy" + Msg Field = "msg" + Severity Field = "severity" + Kind Field = "kind" +) + +func (w WaapEvent) GetVar(varName string) string { + if w.Vars == nil { + return "" + } + if val, ok := w.Vars[varName]; ok { + return val + } + log.Infof("var %s not found. Available variables: %+v", varName, w.Vars) + return "" + +} + +// getters +func (w MatchedRules) GetField(field Field) []interface{} { + ret := make([]interface{}, 0) + for _, rule := range w { + ret = append(ret, rule[field.String()]) + } + return ret +} + +func (w MatchedRules) GetURI() string { + for _, rule := range w { + return rule["uri"].(string) + } + return "" +} + +func (w MatchedRules) GetHash() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["hash"].(string) + } + return "" +} + +func (w MatchedRules) GetVersion() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["version"].(string) + } + return "" +} + +func (w MatchedRules) GetName() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["name"].(string) + } + return "" +} + +func (w MatchedRules) GetMethod() string { + for _, rule := range w { + return rule["method"].(string) + } + return "" +} + +func (w MatchedRules) GetRuleIDs() []int { + ret := make([]int, 0) + for _, rule := range w { + ret = append(ret, rule["id"].(int)) + } + return ret +} + +func (w MatchedRules) Kinds() []string { + ret := make([]string, 0) + for _, rule := range w { + exists := false + for _, val := range ret { + if val == rule["kind"] { + exists = true + break + } + } + if !exists { + ret = append(ret, rule["kind"].(string)) + } + } + return ret +} + +// filters +func (w MatchedRules) ByID(id int) MatchedRules { + waap := MatchedRules{} + + for _, rule := range w { + if rule["id"] == id { + waap = append(waap, rule) + } + } + return waap +} + +func (w MatchedRules) ByKind(kind string) MatchedRules { + waap := MatchedRules{} + for _, rule := range w { + if rule["kind"] == kind { + waap = append(waap, rule) + } + } + return waap +} + +func (w MatchedRules) ByTags(match []string) MatchedRules { + waap := MatchedRules{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + for _, match_tag := range match { + if tag == match_tag { + waap = append(waap, rule) + break + } + } + } + } + return waap +} + +func (w MatchedRules) ByTag(match string) MatchedRules { + waap := MatchedRules{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if tag == match { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w MatchedRules) ByTagRx(rx string) MatchedRules { + waap := MatchedRules{} + re := regexp.MustCompile(rx) + if re == nil { + return waap + } + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + log.Infof("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) + if re.MatchString(tag) { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules { + log.Infof("%s", w) + wap := MatchedRules{} + for _, rule := range w { + if rule["disruptive"] == is { + wap = append(wap, rule) + } + } + log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap)) + + return wap +} + +func (w MatchedRules) BySeverity(severity string) MatchedRules { + wap := MatchedRules{} + for _, rule := range w { + if rule["severity"] == severity { + wap = append(wap, rule) + } + } + log.Infof("BySeverity(%s) -> %d", severity, len(wap)) + return wap +} + +func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules { + wap := MatchedRules{} + for _, rule := range w { + if rule["accuracy"] == accuracy { + wap = append(wap, rule) + } + } + log.Infof("ByAccuracy(%s) -> %d", accuracy, len(wap)) + return wap +} From 07d463f4f002e2ba2f90c43339dff00fb9dc876e Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 10 Nov 2023 17:56:04 +0100 Subject: [PATCH 167/263] up --- cmd/crowdsec/crowdsec.go | 2 +- pkg/acquisition/modules/waap/waap.go | 9 +-------- pkg/waf/loader.go | 10 +++++----- pkg/waf/waap.go | 11 +++++------ pkg/waf/waap_rules_collection.go | 8 -------- 5 files changed, 12 insertions(+), 28 deletions(-) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 39378fab5f8..686d32f2038 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er return nil, fmt.Errorf("while loading scenarios: %w", err) } - if err := waf.LoadWaapRules(); err != nil { + if err := waf.LoadWaapRules(hub); err != nil { return nil, fmt.Errorf("while loading waap rules: %w", err) } diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 9561d895942..92363b2e3d9 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/trace" @@ -159,12 +158,6 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.WaapRunners = make([]WaapRunner, w.config.Routines) - hub, err := cwhub.GetHub() - - if err != nil { - return fmt.Errorf("unable to load hub : %s", err) - } - for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { wafUUID := uuid.New().String() @@ -178,7 +171,7 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { }), WaapRuntime: &wrt, } - err := runner.Init(hub.GetDataDir()) + err := runner.Init(waapCfg.GetDataDir()) if err != nil { return fmt.Errorf("unable to initialize runner : %s", err) } diff --git a/pkg/waf/loader.go b/pkg/waf/loader.go index 5acc136fc7b..ffddea121e5 100644 --- a/pkg/waf/loader.go +++ b/pkg/waf/loader.go @@ -11,11 +11,11 @@ import ( var waapRules map[string]WaapCollectionConfig = make(map[string]WaapCollectionConfig) //FIXME: would probably be better to have a struct for this -func LoadWaapRules() error { - hub, err := cwhub.GetHub() - if err != nil { - return fmt.Errorf("unable to load hub : %s", err) - } +var hub *cwhub.Hub //FIXME: this is a temporary hack to make the hub available in the package + +func LoadWaapRules(hubInstance *cwhub.Hub) error { + + hub = hubInstance for _, hubWafRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { //log.Infof("loading %s", hubWafRuleItem.LocalPath) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 7cede9df2ad..5dfa7a62cab 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -149,11 +149,6 @@ func (wc *WaapConfig) LoadByPath(file string) error { } func (wc *WaapConfig) Load(configName string) error { - hub, err := cwhub.GetHub() - if err != nil { - return fmt.Errorf("unable to load hub : %s", err) - } - waapConfigs := hub.GetItemMap(cwhub.WAAP_CONFIGS) for _, hubWaapConfigItem := range waapConfigs { @@ -164,7 +159,7 @@ func (wc *WaapConfig) Load(configName string) error { continue } wc.Logger.Infof("loading %s", hubWaapConfigItem.LocalPath) - err = wc.LoadByPath(hubWaapConfigItem.LocalPath) + err := wc.LoadByPath(hubWaapConfigItem.LocalPath) if err != nil { return fmt.Errorf("unable to load waap-config %s : %s", hubWaapConfigItem.LocalPath, err) } @@ -174,6 +169,10 @@ func (wc *WaapConfig) Load(configName string) error { return fmt.Errorf("no waap-config found for %s", configName) } +func (wc *WaapConfig) GetDataDir() string { + return hub.GetDataDir() +} + func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret := &WaapRuntimeConfig{} ret.Name = wc.Name diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 7d96549fffd..472cd494c18 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -1,13 +1,11 @@ package waf import ( - "fmt" "os" "path/filepath" "strings" corazatypes "github.com/crowdsecurity/coraza/v3/types" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" log "github.com/sirupsen/logrus" @@ -48,12 +46,6 @@ type RulesDetails struct { var WaapRulesDetails = make(map[int]RulesDetails) func LoadCollection(pattern string) ([]WaapCollection, error) { - - hub, err := cwhub.GetHub() - if err != nil { - return nil, fmt.Errorf("unable to load hub : %s", err) - } - ret := make([]WaapCollection, 0) for _, waapRule := range waapRules { From 6dec8a24bb0b0fb4a0921f01b554f4f52dbdb032 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 14 Nov 2023 10:17:39 +0100 Subject: [PATCH 168/263] update coraza --- go.mod | 17 ++++++++--------- go.sum | 36 ++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index fe208ae97c9..f5c327395e5 100644 --- a/go.mod +++ b/go.mod @@ -77,11 +77,10 @@ require ( github.com/stretchr/testify v1.8.4 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 - golang.org/x/crypto v0.10.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 github.com/xhit/go-simple-mail/v2 v2.16.0 + golang.org/x/crypto v0.15.0 golang.org/x/mod v0.11.0 - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.14.0 google.golang.org/grpc v1.56.1 google.golang.org/protobuf v1.30.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -90,7 +89,7 @@ require ( ) require ( - github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f + github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.27.3 ) @@ -186,7 +185,7 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.2.1 // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -199,10 +198,10 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/term v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.2.0 // indirect golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 4c930935e24..e7e256cf38e 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,10 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f h1:7MgSs0ryJrdGV0f17xBQJPfdD8oo/mqcIQjpqdPo2aA= -github.com/crowdsecurity/coraza/v3 v3.0.0-20230727080316-2348f4b3045f/go.mod h1:YwM+m6iBdUn6P1eQKu+F+83bzkP0AzSEBCcVL//zh9c= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231113100456-9fb1947fe2bf h1:NzG+bC9a1dL9RagY7Rp84CLU4ARLg4EvC7DIP6/czcA= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231113100456-9fb1947fe2bf/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 h1:Kp1sY2PE1H5nbr7xgAQeEWDqDW/o3HNL1rHvcVqzWT4= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= @@ -172,6 +174,7 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= +github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -600,6 +603,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -767,8 +771,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs= github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -867,8 +871,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -944,8 +948,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -964,8 +968,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1031,13 +1035,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1047,8 +1051,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 042d316fab3bc2be2cc20712f50159eebeb7d836 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:58:36 +0100 Subject: [PATCH 169/263] Refact cwhub: remove global hub, func test improvements (#2588) * csConfig.Cscli is always loaded now, configuration paths too * Remove global/singleton hub instance * read {index_path} from config instead of assuming {hub_dir}/.index.json * fix segfault with cscli explain when no parser is installed * cscli: help text * hub download timeout 20 sec * reduce log verbosity * allow func tests with empty hub or pre-download * cscli remove --all --purge --- cmd/crowdsec-cli/hub.go | 9 ++-- cmd/crowdsec-cli/itemcommands.go | 23 +++++++--- cmd/crowdsec-cli/require/require.go | 1 + pkg/cwhub/cwhub.go | 2 +- pkg/cwhub/helpers.go | 2 +- pkg/cwhub/items.go | 17 ++++++++ pkg/hubtest/parser_assert.go | 5 +++ test/bats/01_cscli.bats | 50 +++++++++++----------- test/bats/04_capi.bats | 4 ++ test/bats/05_config_yaml_local.bats | 3 ++ test/bats/07_setup.bats | 20 ++++----- test/bats/20_hub.bats | 17 ++++---- test/bats/20_hub_collections.bats | 13 ++++-- test/bats/20_hub_collections_dep.bats | 9 ++-- test/bats/20_hub_items.bats | 19 ++++---- test/bats/20_hub_parsers.bats | 15 ++++--- test/bats/20_hub_postoverflows.bats | 13 ++++-- test/bats/20_hub_scenarios.bats | 13 ++++-- test/bats/40_cold-logs.bats | 6 ++- test/bats/40_live-ban.bats | 5 +++ test/bats/50_simulation.bats | 5 +++ test/bats/81_alert_context.bats | 3 ++ test/bats/testdata/explain/explain-log.txt | 7 +-- test/lib/config/config-global | 2 +- test/lib/config/config-local | 42 ++++++++++++++++-- 25 files changed, 208 insertions(+), 97 deletions(-) diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index fe0e8b8cfd1..0b61cec4b97 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -12,7 +12,7 @@ import ( ) func NewHubCmd() *cobra.Command { - var cmdHub = &cobra.Command{ + cmdHub := &cobra.Command{ Use: "hub [action]", Short: "Manage hub index", Long: `Hub management @@ -63,7 +63,7 @@ func runHubList(cmd *cobra.Command, args []string) error { } func NewHubListCmd() *cobra.Command { - var cmdHubList = &cobra.Command{ + cmdHubList := &cobra.Command{ Use: "list [-a]", Short: "List all installed configurations", Args: cobra.ExactArgs(0), @@ -95,7 +95,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error { } func NewHubUpdateCmd() *cobra.Command { - var cmdHubUpdate = &cobra.Command{ + cmdHubUpdate := &cobra.Command{ Use: "update", Short: "Download the latest index (catalog of available configurations)", Long: ` @@ -129,6 +129,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { } updated := 0 + log.Infof("Upgrading %s", itemType) for _, item := range items { didUpdate, err := item.Upgrade(force) @@ -146,7 +147,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { } func NewHubUpgradeCmd() *cobra.Command { - var cmdHubUpgrade = &cobra.Command{ + cmdHubUpgrade := &cobra.Command{ Use: "upgrade", Short: "Upgrade all configurations to their latest version", Long: ` diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 3880afc146d..6a8b90996e2 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -61,7 +61,9 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs listHelp: cmdHelp{ example: `cscli parsers list cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +List only enabled parsers unless "-a" or names are specified.`, }, }, "postoverflows": { @@ -91,7 +93,9 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns listHelp: cmdHelp{ example: `cscli postoverflows list cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns + +List only enabled postoverflows unless "-a" or names are specified.`, }, }, "scenarios": { @@ -121,7 +125,9 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing listHelp: cmdHelp{ example: `cscli scenarios list cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing + +List only enabled scenarios unless "-a" or names are specified.`, }, }, "collections": { @@ -151,7 +157,9 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables listHelp: cmdHelp{ example: `cscli collections list cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables + +List only enabled collections unless "-a" or names are specified.`, }, }, } @@ -280,7 +288,12 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e } if all { - items, err := hub.GetInstalledItems(it.name) + getter := hub.GetInstalledItems + if (purge) { + getter = hub.GetAllItems + } + + items, err := getter(it.name) if err != nil { return err } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index 47ad50d0aec..292862e44c4 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -74,6 +74,7 @@ func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg { remote := &cwhub.RemoteHubCfg { Branch: branch, URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", + // URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s", IndexPath: ".index.json", } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index fa59d63da34..74d533c74f8 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -10,5 +10,5 @@ import ( ) var hubClient = &http.Client{ - Timeout: 10 * time.Second, + Timeout: 20 * time.Second, } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 9eeb80088bc..7c6dd62ac98 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -22,7 +22,7 @@ import ( // Install installs the item from the hub, downloading it if needed func (i *Item) Install(force bool, downloadOnly bool) error { if downloadOnly && i.Downloaded && i.UpToDate { - log.Warningf("%s is already downloaded and up-to-date", i.Name) + log.Infof("%s is already downloaded and up-to-date", i.Name) if !force { return nil diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index e9a1b8d4bcf..cbbb30cdae7 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -281,6 +281,23 @@ func (h *Hub) GetItemNames(itemType string) []string { return names } +// GetAllItems returns a slice of all the items, installed or not +func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { + items, ok := h.Items[itemType] + if !ok { + return nil, fmt.Errorf("no %s in the hub index", itemType) + } + + ret := make([]*Item, len(items)) + + idx := 0 + for _, item := range items { + ret[idx] = item + idx++ + } + + return ret, nil +} // GetInstalledItems returns the list of installed items func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { items, ok := h.Items[itemType] diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 620e2645e57..aadf16af7db 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -369,6 +369,11 @@ func LoadParserDump(filepath string) (*ParserResults, error) { } sort.Strings(parsers) + + if len(parsers) == 0 { + return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry") + } + lastParser := parsers[len(parsers)-1] for idx, result := range pdump[lastStage][lastParser] { diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 3e61bd807c7..2662c442395 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -249,36 +249,29 @@ teardown() { # we check for the presence of some objects. There may be others when we # use $PACKAGE_TESTING, so the order is not important. + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli scenarios install crowdsecurity/asterisk_user_enum + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli hub list -o human - assert_line --regexp '^ crowdsecurity/linux' + assert_line --regexp '^ crowdsecurity/whitelists' + assert_line --regexp '^ crowdsecurity/asterisk_user_enum' assert_line --regexp '^ crowdsecurity/sshd' - assert_line --regexp '^ crowdsecurity/dateparse-enrich' - assert_line --regexp '^ crowdsecurity/geoip-enrich' - assert_line --regexp '^ crowdsecurity/sshd-logs' - assert_line --regexp '^ crowdsecurity/syslog-logs' - assert_line --regexp '^ crowdsecurity/ssh-bf' - assert_line --regexp '^ crowdsecurity/ssh-slow-bf' + assert_line --regexp '^ crowdsecurity/rdns' rune -0 cscli hub list -o raw - assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$' - assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$' - assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$' - assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$' - assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$' - assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$' - assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$' - assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$' + assert_line --regexp '^crowdsecurity/whitelists,enabled,.*' + assert_line --regexp '^crowdsecurity/asterisk_user_enum,enabled,.*' + assert_line --regexp '^crowdsecurity/sshd,enabled,.*' + assert_line --regexp '^crowdsecurity/rdns,enabled,.*' rune -0 cscli hub list -o json - rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output) - assert_line 'crowdsecurity/linux' + rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name, .postoverflows[].name' <(output) + assert_line 'crowdsecurity/whitelists' + assert_line 'crowdsecurity/asterisk_user_enum' assert_line 'crowdsecurity/sshd' - assert_line 'crowdsecurity/dateparse-enrich' - assert_line 'crowdsecurity/geoip-enrich' - assert_line 'crowdsecurity/sshd-logs' - assert_line 'crowdsecurity/syslog-logs' - assert_line 'crowdsecurity/ssh-bf' - assert_line 'crowdsecurity/ssh-slow-bf' + assert_line 'crowdsecurity/rdns' } @test "cscli support dump (smoke test)" { @@ -287,8 +280,17 @@ teardown() { } @test "cscli explain" { - rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC" + line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" + + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli collections install crowdsecurity/sshd + + rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC" assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt + + rune -0 cscli parsers remove --all --purge + rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC" + assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry" } @test 'Allow variable expansion and literal $ characters in passwords' { diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index ef933e10c97..04d3a1aa0fa 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -22,6 +22,10 @@ setup() { @test "cscli capi status" { config_enable_capi rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX + rune -1 cscli capi status + assert_stderr --partial "no scenarios installed, abort" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf rune -0 cscli capi status assert_stderr --partial "Loaded credentials from" assert_stderr --partial "Trying to authenticate with username" diff --git a/test/bats/05_config_yaml_local.bats b/test/bats/05_config_yaml_local.bats index 3cc20819bfb..52b2eae5360 100644 --- a/test/bats/05_config_yaml_local.bats +++ b/test/bats/05_config_yaml_local.bats @@ -127,6 +127,9 @@ teardown() { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') echo -e "---\nfilename: ${tmpfile}\nlabels:\n type: syslog\n" >>"${ACQUIS_YAML}" + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli parsers install crowdsecurity/syslog-logs + ./instance-crowdsec start sleep .5 fake_log >>"${tmpfile}" diff --git a/test/bats/07_setup.bats b/test/bats/07_setup.bats index 1748d804072..9e3f5533728 100644 --- a/test/bats/07_setup.bats +++ b/test/bats/07_setup.bats @@ -507,18 +507,16 @@ update-notifier-motd.timer enabled enabled @test "cscli setup install-hub (dry run)" { # it's not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) # we install it rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' assert_output 'dry-run: would install collection crowdsecurity/apache2' # still not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) # same with dependencies rune -0 cscli collections remove --all @@ -528,18 +526,16 @@ update-notifier-motd.timer enabled enabled @test "cscli setup install-hub (dry run: install multiple collections)" { # it's not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) # we install it rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' assert_output 'dry-run: would install collection crowdsecurity/apache2' # still not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/apache2" + rune -0 cscli collections inspect crowdsecurity/apache2 -o json + rune -0 jq -e '.installed == false' <(output) } @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 0a45b08398c..1e475c68067 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -6,8 +6,8 @@ set -u setup_file() { load "../lib/setup_file.sh" ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +20,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -31,6 +30,8 @@ teardown() { #---------- @test "cscli hub list" { + hub_purge_all + # no items rune -0 cscli hub list assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" @@ -49,22 +50,22 @@ teardown() { rune -0 cscli hub list -o raw assert_output --partial 'crowdsecurity/whitelists' assert_output --partial 'crowdsecurity/telnet-bf' - refute_output --partial 'crowdsecurity/linux' + refute_output --partial 'crowdsecurity/iptables' # all items rune -0 cscli hub list -a - assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/linux.*" + assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*" rune -0 cscli hub list -a -o json rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output) rune -0 cscli hub list -a -o raw assert_output --partial 'crowdsecurity/whitelists' assert_output --partial 'crowdsecurity/telnet-bf' - assert_output --partial 'crowdsecurity/linux' + assert_output --partial 'crowdsecurity/iptables' } @test "missing reference in hub index" { - new_hub=$(jq <"$HUB_DIR/.index.json" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') - echo "$new_hub" >"$HUB_DIR/.index.json" + new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli hub list --error assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index b527e39050a..1852dbef2e2 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -8,6 +8,8 @@ setup_file() { ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -31,6 +32,8 @@ teardown() { #---------- @test "cscli collections list" { + hub_purge_all + # no items rune -0 cscli collections list assert_output --partial "COLLECTIONS" @@ -62,7 +65,7 @@ teardown() { } @test "cscli collections list -a" { - expected=$(jq <"$HUB_DIR/.index.json" -r '.collections | length') + expected=$(jq <"$INDEX_PATH" -r '.collections | length') rune -0 cscli collections list -a rune -0 grep -c disabled <(output) @@ -227,6 +230,7 @@ teardown() { rune -1 cscli collections remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + rune -0 cscli collections remove crowdsecurity/sshd --purge rune -0 cscli collections remove crowdsecurity/sshd assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no removal required' @@ -280,6 +284,7 @@ teardown() { assert_stderr --partial "specify at least one collection to upgrade or '--all'" rune -1 cscli collections upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + rune -0 cscli collections remove crowdsecurity/exim --purge rune -1 cscli collections upgrade crowdsecurity/exim assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed" rune -0 cscli collections install crowdsecurity/exim --download-only @@ -290,8 +295,8 @@ teardown() { sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" # add version 0.0 to all collections - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$HUB_DIR/.index.json" + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli collections install crowdsecurity/sshd diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index 18b940339fe..3e8a1111042 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -6,8 +6,8 @@ set -u setup_file() { load "../lib/setup_file.sh" ./instance-data load - HUB_DIR=$(config_get '.config_paths.hub_dir') - export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +20,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -32,8 +31,8 @@ teardown() { @test "cscli collections (dependencies)" { # inject a dependency: smb requires sshd - hub_dep=$(jq <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}') - echo "$hub_dep" >"$HUB_DIR/.index.json" + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}') + echo "$hub_dep" >"$INDEX_PATH" # verify that installing smb brings sshd rune -0 cscli collections install crowdsecurity/smb diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 80e0934c6e6..fcf80cc5062 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -8,6 +8,8 @@ setup_file() { ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -41,10 +42,10 @@ teardown() { # in a lexical vs semver sort. CrowdSec should report the latest version new_hub=$( \ - jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" \ + jq --arg DIGEST "$sha256_empty" <"$INDEX_PATH" \ '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2":{"digest":$DIGEST, "deprecated": false}, "1.10": {"digest":$DIGEST, "deprecated": false}}}}}' \ ) - echo "$new_hub" >"$HUB_DIR/.index.json" + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli collections install crowdsecurity/sshd @@ -57,16 +58,18 @@ teardown() { } @test "hub index with invalid (non semver) version numbers" { + rune -0 cscli collections remove crowdsecurity/sshd --purge + new_hub=$( \ - jq <"$HUB_DIR/.index.json" \ + jq <"$INDEX_PATH" \ '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2.3.4":{"digest":"foo", "deprecated": false}}}}}' \ ) - echo "$new_hub" >"$HUB_DIR/.index.json" + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli collections install crowdsecurity/sshd - - rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics + rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics -o json # XXX: we are on the verbose side here... - assert_stderr --partial "failed to read Hub index: failed to sync items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" + rune -0 jq -r ".msg" <(stderr) + assert_output "failed to read Hub index: failed to sync items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" } diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 544f03aeff2..9283efc1dcb 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -8,6 +8,8 @@ setup_file() { ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -62,7 +63,7 @@ teardown() { } @test "cscli parsers list -a" { - expected=$(jq <"$HUB_DIR/.index.json" -r '.parsers | length') + expected=$(jq <"$INDEX_PATH" -r '.parsers | length') rune -0 cscli parsers list -a rune -0 grep -c disabled <(output) @@ -224,12 +225,13 @@ teardown() { assert_output "0" } -@test "foo cscli parsers remove [parser]..." { +@test "cscli parsers remove [parser]..." { rune -1 cscli parsers remove assert_stderr --partial "specify at least one parser to remove or '--all'" rune -1 cscli parsers remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + rune -0 cscli parsers remove crowdsecurity/whitelists --purge rune -0 cscli parsers remove crowdsecurity/whitelists assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no removal required' @@ -272,7 +274,7 @@ teardown() { @test "cscli parsers remove [parser]... --force" { # remove a parser that belongs to a collection - rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections install crowdsecurity/sshd rune -0 cscli parsers remove crowdsecurity/sshd-logs assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]" assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" @@ -283,6 +285,7 @@ teardown() { assert_stderr --partial "specify at least one parser to upgrade or '--all'" rune -1 cscli parsers upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + rune -0 cscli parsers remove crowdsecurity/pam-logs --purge rune -1 cscli parsers upgrade crowdsecurity/pam-logs assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed" rune -0 cscli parsers install crowdsecurity/pam-logs --download-only @@ -293,8 +296,8 @@ teardown() { sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" # add version 0.0 to all parsers - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$HUB_DIR/.index.json" + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli parsers install crowdsecurity/whitelists diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index 9f766a2ad40..89e97b321d1 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -8,6 +8,8 @@ setup_file() { ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -31,6 +32,8 @@ teardown() { #---------- @test "cscli postoverflows list" { + hub_purge_all + # no items rune -0 cscli postoverflows list assert_output --partial "POSTOVERFLOWS" @@ -62,7 +65,7 @@ teardown() { } @test "cscli postoverflows list -a" { - expected=$(jq <"$HUB_DIR/.index.json" -r '.postoverflows | length') + expected=$(jq <"$INDEX_PATH" -r '.postoverflows | length') rune -0 cscli postoverflows list -a rune -0 grep -c disabled <(output) @@ -232,6 +235,7 @@ teardown() { rune -1 cscli postoverflows remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge rune -0 cscli postoverflows remove crowdsecurity/rdns assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no removal required' @@ -285,6 +289,7 @@ teardown() { assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" rune -1 cscli postoverflows upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + rune -0 cscli postoverflows remove crowdsecurity/discord-crawler-whitelist --purge rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed" rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only @@ -295,8 +300,8 @@ teardown() { sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" # add version 0.0 to all postoverflows - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$HUB_DIR/.index.json" + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli postoverflows install crowdsecurity/rdns diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 7cbbb010453..fab6f5cb802 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -8,6 +8,8 @@ setup_file() { ./instance-data load HUB_DIR=$(config_get '.config_paths.hub_dir') export HUB_DIR + INDEX_PATH=$(config_get '.config_paths.index_path') + export INDEX_PATH CONFIG_DIR=$(config_get '.config_paths.config_dir') export CONFIG_DIR } @@ -20,7 +22,6 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load - hub_purge_all hub_strip_index } @@ -31,6 +32,8 @@ teardown() { #---------- @test "cscli scenarios list" { + hub_purge_all + # no items rune -0 cscli scenarios list assert_output --partial "SCENARIOS" @@ -62,7 +65,7 @@ teardown() { } @test "cscli scenarios list -a" { - expected=$(jq <"$HUB_DIR/.index.json" -r '.scenarios | length') + expected=$(jq <"$INDEX_PATH" -r '.scenarios | length') rune -0 cscli scenarios list -a rune -0 grep -c disabled <(output) @@ -229,6 +232,7 @@ teardown() { rune -1 cscli scenarios remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge rune -0 cscli scenarios remove crowdsecurity/ssh-bf assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no removal required' @@ -282,6 +286,7 @@ teardown() { assert_stderr --partial "specify at least one scenario to upgrade or '--all'" rune -1 cscli scenarios upgrade blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + rune -0 cscli scenarios remove crowdsecurity/vsftpd-bf --purge rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed" rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only @@ -292,8 +297,8 @@ teardown() { sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d" # add version 0.0 to all scenarios - new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') - echo "$new_hub" >"$HUB_DIR/.index.json" + new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})') + echo "$new_hub" >"$INDEX_PATH" rune -0 cscli scenarios install crowdsecurity/ssh-bf diff --git a/test/bats/40_cold-logs.bats b/test/bats/40_cold-logs.bats index 21c0615c724..36220375b87 100644 --- a/test/bats/40_cold-logs.bats +++ b/test/bats/40_cold-logs.bats @@ -11,9 +11,13 @@ fake_log() { setup_file() { load "../lib/setup_file.sh" - # we reset config and data, and only run the daemon once for all the tests in this file ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + ./instance-crowdsec start } diff --git a/test/bats/40_live-ban.bats b/test/bats/40_live-ban.bats index c410cbce5a0..c6b8ddf1563 100644 --- a/test/bats/40_live-ban.bats +++ b/test/bats/40_live-ban.bats @@ -13,6 +13,11 @@ setup_file() { load "../lib/setup_file.sh" # we reset config and data, but run the daemon only in the tests that need it ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + } teardown_file() { diff --git a/test/bats/50_simulation.bats b/test/bats/50_simulation.bats index 0add1e81644..0d29d6bfd52 100644 --- a/test/bats/50_simulation.bats +++ b/test/bats/50_simulation.bats @@ -12,6 +12,11 @@ fake_log() { setup_file() { load "../lib/setup_file.sh" ./instance-data load + + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error + ./instance-crowdsec start } diff --git a/test/bats/81_alert_context.bats b/test/bats/81_alert_context.bats index 6dd6100b9b4..df741f5f99c 100644 --- a/test/bats/81_alert_context.bats +++ b/test/bats/81_alert_context.bats @@ -20,6 +20,9 @@ teardown_file() { setup() { load "../lib/setup.sh" ./instance-data load + cscli collections install crowdsecurity/sshd --error + cscli parsers install crowdsecurity/syslog-logs --error + cscli parsers install crowdsecurity/dateparse-enrich --error } teardown() { diff --git a/test/bats/testdata/explain/explain-log.txt b/test/bats/testdata/explain/explain-log.txt index aae9e8098a4..76247412c5c 100644 --- a/test/bats/testdata/explain/explain-log.txt +++ b/test/bats/testdata/explain/explain-log.txt @@ -2,15 +2,10 @@ line: Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authenticatio ā”œ s00-raw | ā”” šŸŸ¢ crowdsecurity/syslog-logs (+12 ~9) ā”œ s01-parse - | ā”” šŸŸ¢ crowdsecurity/sshd-logs (+8 ~1) - ā”œ s02-enrich - | ā”œ šŸŸ¢ crowdsecurity/dateparse-enrich (+2 ~2) - | ā”œ šŸŸ¢ crowdsecurity/geoip-enrich (+10) - | ā”” šŸŸ¢ crowdsecurity/whitelists (unchanged) + | ā”” šŸŸ¢ crowdsecurity/sshd-logs (+8) ā”œ-------- parser success šŸŸ¢ ā”œ Scenarios ā”œ šŸŸ¢ crowdsecurity/ssh-bf ā”œ šŸŸ¢ crowdsecurity/ssh-bf_user-enum ā”œ šŸŸ¢ crowdsecurity/ssh-slow-bf ā”” šŸŸ¢ crowdsecurity/ssh-slow-bf_user-enum - diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 592a927c2e8..224cc1343be 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -63,7 +63,7 @@ make_init_data() { [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto - mkdir -p "${LOCAL_INIT_DIR}" + mkdir -p "$LOCAL_INIT_DIR" ./instance-db dump "${LOCAL_INIT_DIR}/database" diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 0e2c86692ee..6641d2cfb48 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -101,6 +101,43 @@ config_generate() { ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml" } +preload_hub_items() { + # pre-download everything but don't install anything + # each test can install what it needs + + echo "Downloading collections..." + + #shellcheck disable=SC2046 + "$CSCLI" collections install \ + $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ + --download-only \ + --warning + + #shellcheck disable=SC2046 + "$CSCLI" parsers install \ + $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ + --download-only \ + --warning + + #shellcheck disable=SC2046 + "$CSCLI" scenarios install \ + $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ + --download-only \ + --warning + + #shellcheck disable=SC2046 + "$CSCLI" postoverflows install \ + $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ + --download-only \ + --warning + + # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. + # so we have to delete the links manually, and leave the downloaded files in place + + "$CSCLI" parsers delete --all --error + "$CSCLI" scenarios delete --all --error + "$CSCLI" postoverflows delete --all --error +} make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -118,9 +155,8 @@ make_init_data() { "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto "$CSCLI" --warning hub update - "$CSCLI" --warning collections install crowdsecurity/linux - # the whitelists are installed by the deb & rpm packages, so we test with the same config - "$CSCLI" --warning parsers install crowdsecurity/whitelists + + preload_hub_items mkdir -p "$LOCAL_INIT_DIR" From f8c91d20b0acc81d440fbb55c0fe6091185f95b9 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:20:28 +0100 Subject: [PATCH 170/263] enable CI tests for hub-1.5.6 (#2592) --- .github/workflows/bats.yml | 2 ++ .github/workflows/codeql-analysis.yml | 2 ++ .github/workflows/docker-tests.yml | 2 ++ .github/workflows/go-tests-windows.yml | 2 ++ .github/workflows/go-tests.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index 0ce8cf041ed..fbe5094d184 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -15,12 +15,14 @@ on: push: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - "README.md" pull_request: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - "README.md" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ce94822747d..d0f091e5cbd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,11 +16,13 @@ on: push: branches: - master + - hub-1.5.6 - releases/** pull_request: # The branches below must be a subset of the branches above branches: - master + - hub-1.5.6 - releases/** schedule: - cron: '15 16 * * 2' diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 913c4766238..0589975c6d6 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -4,12 +4,14 @@ on: push: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index aa2116b1c81..cb7d1645ccf 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -4,12 +4,14 @@ on: push: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 2a760d14966..21b9c297bb1 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -9,12 +9,14 @@ on: push: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master + - hub-1.5.6 - releases/** paths-ignore: - 'README.md' From 4a6fd338e0300a7d1cb0f522e24e81d101b2ae7a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:36:07 +0100 Subject: [PATCH 171/263] replace 'timeout' helper with async python script; allow hub preload in func tests; improve item removal (#2591) * replace 'timeout' helper with async python script; allow hub preload in func tests; improve item removal * func tests: cscli hub update/upgrade * docker test update * Update docker entrypoint to disable items with --force The --force flag was not transmitted to cscli, but is required after the hub refact to disable items inside installed collections --- docker/docker_start.sh | 14 ++- docker/test/tests/test_hub_collections.py | 8 +- docker/test/tests/test_hub_scenarios.py | 4 +- pkg/cwhub/enable.go | 29 ++++-- pkg/cwhub/helpers.go | 10 -- pkg/cwhub/items.go | 1 + test/bats/00_wait_for.bats | 71 +++++++++++++ test/bats/01_crowdsec.bats | 55 +++++----- test/bats/02_nolapi.bats | 16 +-- test/bats/03_noagent.bats | 17 ++-- test/bats/04_nocapi.bats | 16 ++- test/bats/05_config_yaml_local.bats | 14 +-- test/bats/13_capi_whitelists.bats | 36 +++++-- test/bats/20_hub.bats | 32 ++++-- test/bats/20_hub_collections.bats | 18 ++-- test/bats/20_hub_items.bats | 23 ++++- test/bats/20_hub_parsers.bats | 19 +++- test/bats/20_hub_postoverflows.bats | 16 ++- test/bats/20_hub_scenarios.bats | 15 ++- test/bats/30_machines_tls.bats | 10 +- test/bats/72_plugin_badconfig.bats | 62 +++++++----- test/bin/wait-for | 116 ++++++++++++++++++++++ test/lib/config/config-global | 49 +++++++++ test/lib/config/config-local | 17 +++- test/lib/setup_file.sh | 1 + 25 files changed, 507 insertions(+), 162 deletions(-) create mode 100644 test/bats/00_wait_for.bats create mode 100755 test/bin/wait-for diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 420b7b8d9f9..b9c35f55289 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -101,19 +101,23 @@ register_bouncer() { # $2 can be install, remove, upgrade # $3 is a list of object names separated by space cscli_if_clean() { + local itemtype="$1" + local action="$2" + local objs=$3 + shift 3 # loop over all objects - for obj in $3; do - if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then - echo "Object $1/$obj is tainted, skipping" + for obj in $objs; do + if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then + echo "Object $itemtype/$obj is tainted, skipping" else # # Too verbose? Only show errors if not in debug mode # if [ "$DEBUG" != "true" ]; then # error_only=--error # fi error_only="" - echo "Running: cscli $error_only $1 $2 \"$obj\"" + echo "Running: cscli $error_only $itemtype $action \"$obj\" $*" # shellcheck disable=SC2086 - cscli $error_only "$1" "$2" "$obj" + cscli $error_only "$itemtype" "$action" "$obj" "$@" fi done } diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index b890bebb9c6..962f8ff8df4 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor): cs.wait_for_log([ # f'*collections install "{it1}"*' # f'*collections install "{it2}"*' - f'*Enabled collections : {it1}*', - f'*Enabled collections : {it2}*', + f'*Enabled collections: {it1}*', + f'*Enabled collections: {it2}*', ]) @@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor): assert it not in items logs = cs.log_lines() # check that there was no attempt to install - assert not any(f'Enabled collections : {it}' in line for line in logs) + assert not any(f'Enabled collections: {it}' in line for line in logs) # already done in bats, prividing here as example of a somewhat complex test @@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): # implicit check for tainted=False assert items[coll]['status'] == 'enabled' cs.wait_for_log([ - f'*Enabled collections : {coll}*', + f'*Enabled collections: {coll}*', ]) scenario = 'crowdsecurity/http-crawl-non_statics' diff --git a/docker/test/tests/test_hub_scenarios.py b/docker/test/tests/test_hub_scenarios.py index a60ede667b8..2a8c3a275f2 100644 --- a/docker/test/tests/test_hub_scenarios.py +++ b/docker/test/tests/test_hub_scenarios.py @@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor): } with crowdsec(flavor=flavor, environment=env) as cs: cs.wait_for_log([ - f'*scenarios install "{it1}*"', - f'*scenarios install "{it2}*"', + f'*scenarios install "{it1}"*', + f'*scenarios install "{it2}"*', "*Starting processing data*" ]) cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 4886899fdc9..b2c02b2bb77 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -76,10 +76,19 @@ func (i *Item) enable() error { // purge removes the actual config file that was downloaded func (i *Item) purge() error { + if !i.Downloaded { + log.Infof("removing %s: not downloaded -- no need to remove", i.Name) + return nil + } + itempath := i.hub.local.HubDir + "/" + i.RemotePath // disable hub file if err := os.Remove(itempath); err != nil { + if os.IsNotExist(err) { + log.Debugf("%s doesn't exist, no need to remove", itempath) + return nil + } return fmt.Errorf("while removing file: %w", err) } @@ -94,17 +103,6 @@ func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not var err error - // already disabled, noop unless purge - if !i.Installed { - if purge { - if err = i.purge(); err != nil { - return err - } - } - - return nil - } - if i.IsLocal() { return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) } @@ -134,6 +132,11 @@ func (i *Item) disable(purge bool, force bool) error { } } + if !i.Installed && !purge { + log.Infof("removing %s: not installed -- no need to remove", i.Name) + return nil + } + syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName) if err != nil { return err @@ -168,6 +171,10 @@ func (i *Item) disable(purge bool, force bool) error { } if err = os.Remove(syml); err != nil { + if os.IsNotExist(err) { + log.Debugf("%s doesn't exist, no need to remove", syml) + return nil + } return fmt.Errorf("while removing symlink: %w", err) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 7c6dd62ac98..2427a8fcb07 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -55,16 +55,6 @@ func (i *Item) Install(force bool, downloadOnly bool) error { func (i *Item) Remove(purge bool, forceAction bool) (bool, error) { removed := false - if !i.Downloaded { - log.Infof("removing %s: not downloaded -- no removal required", i.Name) - return false, nil - } - - if !i.Installed && !purge { - log.Infof("removing %s: already uninstalled", i.Name) - return false, nil - } - if err := i.disable(purge, forceAction); err != nil { return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index cbbb30cdae7..280fcec3e26 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -298,6 +298,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { return ret, nil } + // GetInstalledItems returns the list of installed items func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { items, ok := h.Items[itemType] diff --git a/test/bats/00_wait_for.bats b/test/bats/00_wait_for.bats new file mode 100644 index 00000000000..ffc6802d9bc --- /dev/null +++ b/test/bats/00_wait_for.bats @@ -0,0 +1,71 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" +} + +setup() { + load "../lib/setup.sh" +} + +@test "run a command and capture its stdout" { + run -0 wait-for seq 1 3 + assert_output - <<-EOT + 1 + 2 + 3 + EOT +} + +@test "run a command and capture its stderr" { + rune -0 wait-for sh -c 'seq 1 3 >&2' + assert_stderr - <<-EOT + 1 + 2 + 3 + EOT +} + +@test "run a command until a pattern is found in stdout" { + run -0 wait-for --out "1[12]0" seq 1 200 + assert_line --index 0 "1" + assert_line --index -1 "110" + refute_line "111" +} + +@test "run a command until a pattern is found in stderr" { + rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2' + assert_stderr - <<-EOT + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + EOT +} + +@test "run a command with timeout (no match)" { + # when the process is terminated without a match, it returns + # 256 - 15 (SIGTERM) = 241 + rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2' + assert_line 1 + # there may be more, but we don't care +} + +@test "run a command with timeout (match)" { + # when the process is terminated with a match, return code is 128 + rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10' + assert_output - <<-EOT + 1 + 2 + EOT +} + diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 40dae750ea1..7bcc35b1d80 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -24,28 +24,22 @@ teardown() { #---------- @test "crowdsec (usage)" { - rune -0 timeout 2s "${CROWDSEC}" -h - assert_stderr_line --regexp "Usage of .*:" - - rune -0 timeout 2s "${CROWDSEC}" --help - assert_stderr_line --regexp "Usage of .*:" + rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h + rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help } @test "crowdsec (unknown flag)" { - rune -2 timeout 2s "${CROWDSEC}" --foobar - assert_stderr_line "flag provided but not defined: -foobar" - assert_stderr_line --regexp "Usage of .*" + rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar } @test "crowdsec (unknown argument)" { - rune -2 timeout 2s "${CROWDSEC}" trololo - assert_stderr_line "argument provided but not defined: trololo" - assert_stderr_line --regexp "Usage of .*" + rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo } @test "crowdsec (no api and no agent)" { - rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs - assert_stderr_line --partial "You must run at least the API Server or crowdsec" + rune -0 wait-for \ + --err "You must run at least the API Server or crowdsec" \ + "${CROWDSEC}" -no-api -no-cs } @test "crowdsec - print error on exit" { @@ -57,18 +51,20 @@ teardown() { @test "crowdsec - default logging configuration (empty/missing common section)" { config_set '.common={}' - rune -124 timeout 2s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" refute_output - assert_stderr --partial "Starting processing data" config_set 'del(.common)' - rune -124 timeout 2s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" refute_output - assert_stderr --partial "Starting processing data" } @test "CS_LAPI_SECRET not strong enough" { - CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}" + CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}" assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough" } @@ -138,8 +134,8 @@ teardown() { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') rm -f "$ACQUIS_YAML" - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr_line --partial "acquis.yaml: no such file or directory" + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "acquis.yaml: no such file or directory" } @test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" { @@ -151,7 +147,7 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" # check warning assert_stderr --partial "no acquisition file found" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" @@ -167,13 +163,15 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" # check warning assert_stderr --partial "no acquisition_path or acquisition_dir specified" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { + config_set '.common.log_media="stdout"' + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') config_set '.crowdsec_service.acquisition_path=""' @@ -181,13 +179,15 @@ teardown() { mkdir -p "$ACQUIS_DIR" mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml - rune -124 timeout 2s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" # now, if foo.yaml is empty instead, there won't be valid datasources. cat /dev/null >"$ACQUIS_DIR"/foo.yaml - rune -1 timeout 2s "${CROWDSEC}" + rune -1 wait-for "${CROWDSEC}" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @@ -212,9 +212,10 @@ teardown() { type: syslog EOT - rune -124 timeout 2s env PATH='' "${CROWDSEC}" #shellcheck disable=SC2016 - assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' + rune -0 wait-for \ + --err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \ + env PATH='' "${CROWDSEC}" # if all datasources are disabled, crowdsec should exit @@ -222,7 +223,7 @@ teardown() { rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' - rune -1 timeout 2s env PATH='' "${CROWDSEC}" + rune -1 wait-for env PATH='' "${CROWDSEC}" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } diff --git a/test/bats/02_nolapi.bats b/test/bats/02_nolapi.bats index a434e8a6d84..dec94a86b19 100644 --- a/test/bats/02_nolapi.bats +++ b/test/bats/02_nolapi.bats @@ -24,21 +24,23 @@ teardown() { #---------- @test "test without -no-api flag" { - rune -124 timeout 2s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "CrowdSec Local API listening" \ + "${CROWDSEC}" } @test "crowdsec should not run without LAPI (-no-api flag)" { - # really needs 4 secs on slow boxes - rune -1 timeout 4s "${CROWDSEC}" -no-api + config_set '.common.log_media="stdout"' + rune -1 wait-for "${CROWDSEC}" -no-api } @test "crowdsec should not run without LAPI (no api.server in configuration file)" { config_disable_lapi config_log_stderr - # really needs 4 secs on slow boxes - rune -1 timeout 4s "${CROWDSEC}" - assert_stderr --partial "crowdsec local API is disabled" + rune -0 wait-for \ + --err "crowdsec local API is disabled" \ + "${CROWDSEC}" } @test "capi status shouldn't be ok without api.server" { diff --git a/test/bats/03_noagent.bats b/test/bats/03_noagent.bats index a91737f99b5..e75e375ad1c 100644 --- a/test/bats/03_noagent.bats +++ b/test/bats/03_noagent.bats @@ -23,20 +23,25 @@ teardown() { #---------- @test "with agent: test without -no-cs flag" { - rune -124 timeout 2s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" } @test "no agent: crowdsec LAPI should run (-no-cs flag)" { - rune -124 timeout 2s "${CROWDSEC}" -no-cs + config_set '.common.log_media="stdout"' + rune -0 wait-for \ + --err "CrowdSec Local API listening" \ + "${CROWDSEC}" -no-cs } @test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" { config_disable_agent config_log_stderr - rune -124 timeout 2s "${CROWDSEC}" - - assert_stderr --partial "crowdsec agent is disabled" + rune -0 wait-for \ + --err "crowdsec agent is disabled" \ + "${CROWDSEC}" } @test "no agent: cscli config show" { diff --git a/test/bats/04_nocapi.bats b/test/bats/04_nocapi.bats index 23994c43e17..234db182a53 100644 --- a/test/bats/04_nocapi.bats +++ b/test/bats/04_nocapi.bats @@ -25,16 +25,17 @@ teardown() { @test "without capi: crowdsec LAPI should run without capi (-no-capi flag)" { config_set '.common.log_media="stdout"' - rune -124 timeout 1s "${CROWDSEC}" -no-capi - assert_stderr --partial "Communication with CrowdSec Central API disabled from args" + rune -0 wait-for \ + --err "Communication with CrowdSec Central API disabled from args" \ + "${CROWDSEC}" -no-capi } @test "without capi: crowdsec LAPI should still work" { config_disable_capi config_set '.common.log_media="stdout"' - rune -124 timeout 1s "${CROWDSEC}" - # from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. - assert_stderr --partial "push and pull to Central API disabled" + rune -0 wait-for \ + --err "push and pull to Central API disabled" \ + "${CROWDSEC}" } @test "without capi: cscli capi status -> fail" { @@ -47,10 +48,7 @@ teardown() { @test "no capi: cscli config show" { config_disable_capi rune -0 cscli config show -o human - assert_output --partial "Global:" - assert_output --partial "cscli:" - assert_output --partial "Crowdsec:" - assert_output --partial "Local API Server:" + assert_output --regexp "Global:.*Crowdsec.*cscli:.*Local API Server:" } @test "no agent: cscli config backup" { diff --git a/test/bats/05_config_yaml_local.bats b/test/bats/05_config_yaml_local.bats index 52b2eae5360..6f8b5c28bd7 100644 --- a/test/bats/05_config_yaml_local.bats +++ b/test/bats/05_config_yaml_local.bats @@ -56,28 +56,28 @@ teardown() { # disable the agent or we'll need to patch api client credentials too rune -0 config_disable_agent ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8080 + rune -0 wait-for-port -q 8080 ./instance-crowdsec stop - rune -1 ./bin/wait-for-port -q 8080 + rune -1 wait-for-port -q 8080 echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local" ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8083 - rune -1 ./bin/wait-for-port -q 8080 + rune -0 wait-for-port -q 8083 + rune -1 wait-for-port -q 8080 ./instance-crowdsec stop rm -f "${CONFIG_YAML}.local" ./instance-crowdsec start - rune -1 ./bin/wait-for-port -q 8083 - rune -0 ./bin/wait-for-port -q 8080 + rune -1 wait-for-port -q 8083 + rune -0 wait-for-port -q 8080 } @test "local_api_credentials.yaml.local" { rune -0 config_disable_agent echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local" ./instance-crowdsec start - rune -0 ./bin/wait-for-port -q 8083 + rune -0 wait-for-port -q 8083 rune -1 cscli decisions list echo "{'url':'http://127.0.0.1:8083'}" >"${LOCAL_API_CREDENTIALS}.local" diff --git a/test/bats/13_capi_whitelists.bats b/test/bats/13_capi_whitelists.bats index 491de649874..61d0e641c94 100644 --- a/test/bats/13_capi_whitelists.bats +++ b/test/bats/13_capi_whitelists.bats @@ -18,6 +18,7 @@ setup() { load "../lib/setup.sh" load "../lib/bats-file/load.bash" ./instance-data load + config_set '.common.log_media="stdout"' config_set '.api.server.capi_whitelists_path=strenv(CAPI_WHITELISTS_YAML)' } @@ -28,38 +29,51 @@ teardown() { #---------- @test "capi_whitelists: file missing" { - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist" + rune -0 wait-for \ + --err "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist" \ + "${CROWDSEC}" } @test "capi_whitelists: error on open" { echo > "$CAPI_WHITELISTS_YAML" chmod 000 "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied" + if is_package_testing; then + rune -0 wait-for \ + --err "while parsing capi whitelist file .*: empty file" \ + "${CROWDSEC}" + else + rune -0 wait-for \ + --err "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied" \ + "${CROWDSEC}" + fi } @test "capi_whitelists: empty file" { echo > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file" \ + "${CROWDSEC}" } @test "capi_whitelists: empty lists" { echo '{"ips": [], "cidrs": []}' > "$CAPI_WHITELISTS_YAML" - rune -124 timeout 1s "${CROWDSEC}" + rune -0 wait-for \ + --err "Starting processing data" \ + "${CROWDSEC}" } @test "capi_whitelists: bad ip" { echo '{"ips": ["blahblah"], "cidrs": []}' > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah" \ + "${CROWDSEC}" } @test "capi_whitelists: bad cidr" { echo '{"ips": [], "cidrs": ["blahblah"]}' > "$CAPI_WHITELISTS_YAML" - rune -1 timeout 1s "${CROWDSEC}" - assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah" + rune -0 wait-for \ + --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah" \ + "${CROWDSEC}" } @test "capi_whitelists: file with ip and cidr values" { diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 1e475c68067..c2dd2795964 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -72,16 +72,32 @@ teardown() { } @test "cscli hub update" { - #XXX: todo - : + rm -f "$INDEX_PATH" + rune -0 cscli hub update + assert_stderr --partial "Wrote index to $INDEX_PATH" + rune -0 cscli hub update + assert_stderr --partial "hub index is up to date" } @test "cscli hub upgrade" { - #XXX: todo - : -} + rune -0 cscli hub upgrade + assert_stderr --partial "Upgrading parsers" + assert_stderr --partial "Upgraded 0 parsers" + assert_stderr --partial "Upgrading postoverflows" + assert_stderr --partial "Upgraded 0 postoverflows" + assert_stderr --partial "Upgrading scenarios" + assert_stderr --partial "Upgraded 0 scenarios" + assert_stderr --partial "Upgrading collections" + assert_stderr --partial "Upgraded 0 collections" -@test "cscli hub upgrade --force" { - #XXX: todo - : + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli hub upgrade + assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date" + + rune -0 cscli hub upgrade --force + assert_stderr --partial "crowdsecurity/syslog-logs: overwrite" + assert_stderr --partial "crowdsecurity/syslog-logs: updated" + assert_stderr --partial "Upgraded 1 parsers" + # this is used by the cron script to know if the hub was updated + assert_output --partial "updated crowdsecurity/syslog-logs" } diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 1852dbef2e2..9d0f24ba4c9 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -187,7 +187,6 @@ teardown() { # one item, json rune -0 cscli collections inspect crowdsecurity/sshd -o json rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) - # XXX: .installed is missing -- not false assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]' # one item, raw @@ -220,7 +219,7 @@ teardown() { rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw assert_output --partial 'crowdsecurity/sshd' assert_output --partial 'crowdsecurity/smb' - run -1 grep -c 'Current metrics:' <(output) + rune -1 grep -c 'Current metrics:' <(output) assert_output "0" } @@ -230,16 +229,23 @@ teardown() { rune -1 cscli collections remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in collections" - rune -0 cscli collections remove crowdsecurity/sshd --purge + rune -0 cscli collections install crowdsecurity/sshd --download-only rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no removal required' + assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' - rune -0 cscli collections install crowdsecurity/sshd --download-only + rune -0 cscli collections install crowdsecurity/sshd rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial 'removing crowdsecurity/sshd: already uninstalled' + assert_stderr --partial 'Removed crowdsecurity/sshd' + rune -0 cscli collections remove crowdsecurity/sshd --purge assert_stderr --partial 'Removed source file [crowdsecurity/sshd]' + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove' + + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove' + # install, then remove, check files rune -0 cscli collections install crowdsecurity/sshd assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index fcf80cc5062..0ac3d1c8d3d 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -70,6 +70,27 @@ teardown() { rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics -o json # XXX: we are on the verbose side here... rune -0 jq -r ".msg" <(stderr) - assert_output "failed to read Hub index: failed to sync items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" + assert_output --regexp "failed to read Hub index: failed to sync items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" } +@test "removing or purging an item already removed by hand" { + rune -0 cscli parsers install crowdsecurity/syslog-logs + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -r '.local_path' <(output) + rune -0 rm "$(output)" + + rune -0 cscli parsers remove crowdsecurity/syslog-logs --debug + assert_stderr --partial "Removed crowdsecurity/syslog-logs" + + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -r '.path' <(output) + rune -0 rm "$HUB_DIR/$(output)" + + rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge + assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove" + + rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force + rune -0 cscli collections remove crowdsecurity/linux --all --error --purge --force + refute_output + refute_stderr +} diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index 9283efc1dcb..c17b63315f0 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -32,6 +32,8 @@ teardown() { #---------- @test "cscli parsers list" { + hub_purge_all + # no items rune -0 cscli parsers list assert_output --partial "PARSERS" @@ -221,7 +223,7 @@ teardown() { rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw assert_output --partial 'crowdsecurity/sshd-logs' assert_output --partial 'crowdsecurity/whitelists' - run -1 grep -c 'Current metrics:' <(output) + rune -1 grep -c 'Current metrics:' <(output) assert_output "0" } @@ -231,16 +233,23 @@ teardown() { rune -1 cscli parsers remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" - rune -0 cscli parsers remove crowdsecurity/whitelists --purge + rune -0 cscli parsers install crowdsecurity/whitelists --download-only rune -0 cscli parsers remove crowdsecurity/whitelists - assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no removal required' + assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" - rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers install crowdsecurity/whitelists rune -0 cscli parsers remove crowdsecurity/whitelists - assert_stderr --partial 'removing crowdsecurity/whitelists: already uninstalled' + assert_stderr --partial "Removed crowdsecurity/whitelists" + rune -0 cscli parsers remove crowdsecurity/whitelists --purge assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]' + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove" + + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove' + # install, then remove, check files rune -0 cscli parsers install crowdsecurity/whitelists assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index 89e97b321d1..23c2ef20bec 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -191,7 +191,6 @@ teardown() { # one item, json rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) - # XXX: .installed is missing -- not false assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]' # one item, raw @@ -235,16 +234,23 @@ teardown() { rune -1 cscli postoverflows remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" - rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no removal required' + assert_stderr --partial "removing crowdsecurity/rdns: not installed -- no need to remove" - rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows install crowdsecurity/rdns rune -0 cscli postoverflows remove crowdsecurity/rdns - assert_stderr --partial 'removing crowdsecurity/rdns: already uninstalled' + assert_stderr --partial 'Removed crowdsecurity/rdns' + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge assert_stderr --partial 'Removed source file [crowdsecurity/rdns]' + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove' + + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove' + # install, then remove, check files rune -0 cscli postoverflows install crowdsecurity/rdns assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index fab6f5cb802..8e6d2172563 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -232,16 +232,23 @@ teardown() { rune -1 cscli scenarios remove blahblah/blahblah assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" - rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no removal required' + assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" - rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios install crowdsecurity/ssh-bf rune -0 cscli scenarios remove crowdsecurity/ssh-bf - assert_stderr --partial 'removing crowdsecurity/ssh-bf: already uninstalled' + assert_stderr --partial "Removed crowdsecurity/ssh-bf" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]' + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove" + + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove' + # install, then remove, check files rune -0 cscli scenarios install crowdsecurity/ssh-bf assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" diff --git a/test/bats/30_machines_tls.bats b/test/bats/30_machines_tls.bats index 121cdecdf1b..535435336ba 100644 --- a/test/bats/30_machines_tls.bats +++ b/test/bats/30_machines_tls.bats @@ -78,15 +78,17 @@ teardown() { @test "missing key_file" { config_set '.api.server.tls.key_file=""' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "missing TLS key file" + rune -0 wait-for \ + --err "missing TLS key file" \ + "${CROWDSEC}" } @test "missing cert_file" { config_set '.api.server.tls.cert_file=""' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "missing TLS cert file" + rune -0 wait-for \ + --err "missing TLS cert file" \ + "${CROWDSEC}" } @test "invalid OU for agent" { diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index 4f325b0f933..c9a69b9fcb0 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -27,7 +27,7 @@ setup() { teardown() { ./instance-crowdsec stop rm -f "${PLUGIN_DIR}"/badname - chmod go-w "${PLUGIN_DIR}"/notification-http + chmod go-w "${PLUGIN_DIR}"/notification-http || true } #---------- @@ -35,36 +35,41 @@ teardown() { @test "misconfigured plugin, only user is empty" { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \ + "${CROWDSEC}" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \ + "${CROWDSEC}" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" \ + "${CROWDSEC}" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" \ + "${CROWDSEC}" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" \ + "${CROWDSEC}" } @test "duplicate notification config" { @@ -75,48 +80,55 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["slack_default"]' # the slack plugin may fail or not, but we just need the logs config_set '.common.log_media="stdout"' - rune timeout 2s "${CROWDSEC}" - assert_stderr --partial "notification 'email_default' is defined multiple times" + rune wait-for \ + --err "notification 'email_default' is defined multiple times" \ + "${CROWDSEC}" } @test "bad plugin permission (group writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" \ + "${CROWDSEC}" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" \ + "${CROWDSEC}" } @test "config.yaml: missing .plugin_config section" { config_set 'del(.plugin_config)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but the plugin_config section is missing in the configuration" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but the plugin_config section is missing in the configuration" \ + "${CROWDSEC}" } @test "config.yaml: missing config_paths.notification_dir" { config_set 'del(.config_paths.notification_dir)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but config_paths.notification_dir is not defined" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but config_paths.notification_dir is not defined" \ + "${CROWDSEC}" } @test "config.yaml: missing config_paths.plugin_dir" { config_set 'del(.config_paths.plugin_dir)' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" + rune -0 wait-for \ + --err "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" \ + "${CROWDSEC}" } @test "unable to run plugin broker: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' - rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + rune -0 wait-for \ + --err "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" \ + "${CROWDSEC}" } diff --git a/test/bin/wait-for b/test/bin/wait-for new file mode 100755 index 00000000000..6c6fdd5ce2b --- /dev/null +++ b/test/bin/wait-for @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import os +import re +import signal +import sys + +DEFAULT_TIMEOUT = 30 + +# TODO: signal handler to terminate spawned process group when wait-for is killed +# TODO: better return codes esp. when matches are found +# TODO: multiple patterns (multiple out, err, both) +# TODO: print unmatched patterns + + +async def terminate(p): + # Terminate the process group (shell, crowdsec plugins) + try: + os.killpg(os.getpgid(p.pid), signal.SIGTERM) + except ProcessLookupError: + pass + + +async def monitor(cmd, args, want_out, want_err, timeout): + """Monitor a process and terminate it if a pattern is matched in stdout or stderr. + + Args: + cmd: The command to run. + args: A list of arguments to pass to the command. + stdout: A regular expression pattern to search for in stdout. + stderr: A regular expression pattern to search for in stderr. + timeout: The maximum number of seconds to wait for the process to terminate. + + Returns: + The exit code of the process. + """ + + status = None + + async def read_stream(p, stream, outstream, pattern): + nonlocal status + if stream is None: + return + while True: + line = await stream.readline() + if line: + line = line.decode('utf-8') + outstream.write(line) + if pattern and pattern.search(line): + await terminate(process) + # this is nasty. + # if we timeout, we want to return a different exit code + # in case of a match, so that the caller can tell + # if the application was still running. + # XXX: still not good for match found, but return code != 0 + if timeout != DEFAULT_TIMEOUT: + status = 128 + else: + status = 0 + break + else: + break + + process = await asyncio.create_subprocess_exec( + cmd, + *args, + # capture stdout + stdout=asyncio.subprocess.PIPE, + # capture stderr + stderr=asyncio.subprocess.PIPE, + # disable buffering + bufsize=0, + # create a new process group + # (required to kill child processes when cmd is a shell) + preexec_fn=os.setsid) + + out_regex = re.compile(want_out) if want_out else None + err_regex = re.compile(want_err) if want_err else None + + # Apply a timeout + try: + await asyncio.wait_for( + asyncio.wait([ + asyncio.create_task(process.wait()), + asyncio.create_task(read_stream(process, process.stdout, sys.stdout, out_regex)), + asyncio.create_task(read_stream(process, process.stderr, sys.stderr, err_regex)) + ]), timeout) + if status is None: + status = process.returncode + except asyncio.TimeoutError: + await terminate(process) + status = 241 + + # Return the same exit code, stdout and stderr as the spawned process + return status + + +async def main(): + parser = argparse.ArgumentParser( + description='Monitor a process and terminate it if a pattern is matched in stdout or stderr.') + parser.add_argument('cmd', help='The command to run.') + parser.add_argument('args', nargs=argparse.REMAINDER, help='A list of arguments to pass to the command.') + parser.add_argument('--out', default='', help='A regular expression pattern to search for in stdout.') + parser.add_argument('--err', default='', help='A regular expression pattern to search for in stderr.') + parser.add_argument('--timeout', type=float, default=DEFAULT_TIMEOUT) + args = parser.parse_args() + + exit_code = await monitor(args.cmd, args.args, args.out, args.err, args.timeout) + + sys.exit(exit_code) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 224cc1343be..46960bcab5a 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -38,6 +38,8 @@ DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}" export DATA_DIR CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}" export CONFIG_DIR +HUB_DIR="${CONFIG_DIR}/hub" +export HUB_DIR if [[ $(uname) == "OpenBSD" ]]; then TAR=gtar @@ -52,6 +54,51 @@ remove_init_data() { # we need a separate function for initializing config when testing package # because we want to test the configuration as well +preload_hub_items() { + # pre-download everything but don't install anything + # each test can install what it needs + + echo "Purging existing hub..." + + "$CSCLI" parsers delete --all --error --purge --force + "$CSCLI" scenarios delete --all --error --purge --force + "$CSCLI" postoverflows delete --all --error --purge --force + "$CSCLI" collections delete --all --error --purge --force + + echo "Pre-downloading hub content..." + + #shellcheck disable=SC2046 + "$CSCLI" collections install \ + $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" parsers install \ + $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" scenarios install \ + $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ + --download-only \ + --error + + #shellcheck disable=SC2046 + "$CSCLI" postoverflows install \ + $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ + --download-only \ + --error + + # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. + # so we have to delete the links manually, and leave the downloaded files in place + + "$CSCLI" parsers delete --all --error + "$CSCLI" scenarios delete --all --error + "$CSCLI" postoverflows delete --all --error +} + make_init_data() { ./bin/assert-crowdsec-not-running || die "Cannot create fixture data." @@ -61,6 +108,8 @@ make_init_data() { # when installed packages are always using sqlite, so no need to regenerate # local credz for sqlite + preload_hub_items + [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto mkdir -p "$LOCAL_INIT_DIR" diff --git a/test/lib/config/config-local b/test/lib/config/config-local index 6641d2cfb48..625e6e5ce5c 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -105,31 +105,38 @@ preload_hub_items() { # pre-download everything but don't install anything # each test can install what it needs - echo "Downloading collections..." + echo "Purging existing hub..." + + "$CSCLI" parsers delete --all --error --purge --force + "$CSCLI" scenarios delete --all --error --purge --force + "$CSCLI" postoverflows delete --all --error --purge --force + "$CSCLI" collections delete --all --error --purge --force + + echo "Pre-downloading hub content..." #shellcheck disable=SC2046 "$CSCLI" collections install \ $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \ --download-only \ - --warning + --error #shellcheck disable=SC2046 "$CSCLI" parsers install \ $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \ --download-only \ - --warning + --error #shellcheck disable=SC2046 "$CSCLI" scenarios install \ $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \ --download-only \ - --warning + --error #shellcheck disable=SC2046 "$CSCLI" postoverflows install \ $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \ --download-only \ - --warning + --error # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. # so we have to delete the links manually, and leave the downloaded files in place diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 7d298418adf..256abbbc91b 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -20,6 +20,7 @@ eval "$(debug)" # Allow tests to use relative paths for helper scripts. # shellcheck disable=SC2164 cd "${TEST_DIR}" +export PATH="${TEST_DIR}/bin:${PATH}" # complain if there's a crowdsec running system-wide or leftover from a previous test ./bin/assert-crowdsec-not-running From 056c979455a81d18e5ace2e03ae125ca27319878 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 15 Nov 2023 15:08:57 +0100 Subject: [PATCH 172/263] add support for labels to waap rules --- pkg/waf/waap_rules_collection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 472cd494c18..0b4caccdcea 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -29,6 +29,8 @@ type WaapCollectionConfig struct { SecLangRules []string `yaml:"seclang_rules"` Rules []waap_rule.CustomRule `yaml:"rules"` + Labels map[string]interface{} `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow + Data interface{} `yaml:"data"` //Ignore it hash string `yaml:"-"` version string `yaml:"-"` From 79d019f9a27dd0fa80e11198feb6394dd22799fe Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:59:30 +0100 Subject: [PATCH 173/263] Refact cwhub / sort cscli output, case insensitive (#2593) * dead code: unknown localVersion now defaults to "?" * skip type declaration; whitespace * sync: next item if invalid cpath * func tests for install --force and --ignore * shorter test names * sort cscli output, with tests * cscli: refact hub sort code --- cmd/crowdsec-cli/itemcommands.go | 2 +- cmd/crowdsec-cli/items.go | 41 +++++++++++---------- cmd/crowdsec-cli/utils_table.go | 5 ++- pkg/cwhub/cwhub_test.go | 4 +-- pkg/cwhub/enable.go | 9 +++-- pkg/cwhub/items.go | 10 ++++++ pkg/cwhub/sync.go | 7 ++-- test/bats/20_hub_collections.bats | 48 ++++++++++++++++++++----- test/bats/20_hub_parsers.bats | 52 +++++++++++++++++++++------ test/bats/20_hub_postoverflows.bats | 55 ++++++++++++++++++++++------- test/bats/20_hub_scenarios.bats | 48 ++++++++++++++++++++----- 11 files changed, 205 insertions(+), 76 deletions(-) diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 6a8b90996e2..f93e3b0b9b2 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -289,7 +289,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e if all { getter := hub.GetInstalledItems - if (purge) { + if purge { getter = hub.GetAllItems } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 6d84582dbed..8104b1ed48d 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "sort" "strings" "gopkg.in/yaml.v3" @@ -15,7 +14,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) { +// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name +func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) { itemNames := hub.GetItemNames(itemType) notExist := []string{} @@ -37,26 +37,32 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b installedOnly = false } - if installedOnly { - installed := []string{} - for _, item := range itemNames { - if hub.GetItem(itemType, item).Installed { - installed = append(installed, item) - } + items := make([]*cwhub.Item, 0, len(itemNames)) + + for _, itemName := range itemNames { + item := hub.GetItem(itemType, itemName) + if installedOnly && !item.Installed { + continue } - return installed, nil + + items = append(items, item) } - return itemNames, nil + + cwhub.SortItemSlice(items) + + return items, nil } +// XXX: too complex, should be two functions (itemtypes array and args are not used together) func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { - items := make(map[string][]string) + items := make(map[string][]*cwhub.Item) + for _, itemType := range itemTypes { selected, err := selectItems(hub, itemType, args, !all) if err != nil { return err } - sort.Strings(selected) + items[itemType] = selected } @@ -79,8 +85,8 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, for _, itemType := range itemTypes { // empty slice in case there are no items of this type hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) - for i, itemName := range items[itemType] { - item := hub.GetItem(itemType, itemName) + + for i, item := range items[itemType] { status, emo := item.Status() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, @@ -110,13 +116,10 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, return fmt.Errorf("failed to write header: %s", err) } } + for _, itemType := range itemTypes { - for _, itemName := range items[itemType] { - item := hub.GetItem(itemType, itemName) + for _, item := range items[itemType] { status, _ := item.Status() - if item.LocalVersion == "" { - item.LocalVersion = "n/a" - } row := []string{ item.Name, status, diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 0336f40a1ee..67a7a4ec547 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -11,14 +11,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, itemNames []string) { +func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, items []*cwhub.Item) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - for itemName := range itemNames { - item := hub.GetItem(itemType, itemNames[itemName]) + for _, item := range items { status, emo := item.Status() t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index af19b93ee7c..fd95dd2eba0 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -60,15 +60,13 @@ func testHub(t *testing.T, update bool) *Hub { os.RemoveAll(tmpDir) }) - var hub *Hub - remote := &RemoteHubCfg{ Branch: "master", URLTemplate: mockURLTemplate, IndexPath: ".index.json", } - hub, err = NewHub(local, remote, update) + hub, err := NewHub(local, remote, update) require.NoError(t, err) return hub diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index b2c02b2bb77..abe1d5e21b3 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -89,6 +89,7 @@ func (i *Item) purge() error { log.Debugf("%s doesn't exist, no need to remove", itempath) return nil } + return fmt.Errorf("while removing file: %w", err) } @@ -101,8 +102,6 @@ func (i *Item) purge() error { // disable removes the symlink to the downloaded content, also removes the content if purge is true func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not - var err error - if i.IsLocal() { return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) } @@ -124,7 +123,7 @@ func (i *Item) disable(purge bool, force bool) error { } if removeSub { - if err = sub.disable(purge, force); err != nil { + if err := sub.disable(purge, force); err != nil { return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } else { @@ -170,7 +169,7 @@ func (i *Item) disable(purge bool, force bool) error { return fmt.Errorf("%s isn't managed by hub", i.Name) } - if err = os.Remove(syml); err != nil { + if err := os.Remove(syml); err != nil { if os.IsNotExist(err) { log.Debugf("%s doesn't exist, no need to remove", syml) return nil @@ -184,7 +183,7 @@ func (i *Item) disable(purge bool, force bool) error { i.Installed = false if purge { - if err = i.purge(); err != nil { + if err := i.purge(); err != nil { return err } } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 280fcec3e26..8e3063a9daa 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -3,6 +3,8 @@ package cwhub import ( "encoding/json" "fmt" + "sort" + "strings" "github.com/Masterminds/semver/v3" "github.com/enescakir/emoji" @@ -291,6 +293,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { ret := make([]*Item, len(items)) idx := 0 + for _, item := range items { ret[idx] = item idx++ @@ -332,3 +335,10 @@ func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { return retStr, nil } + +// SortItemSlice sorts a slice of items by name, case insensitive +func SortItemSlice(items []*Item) { + sort.Slice(items, func(i, j int) bool { + return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) + }) +} diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index fe47342cabc..30336fa91c1 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -150,10 +150,8 @@ func sortedVersions(raw []string) ([]string, error) { } func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { - var ( - local bool - hubpath string - ) + local := false + hubpath := "" if err != nil { log.Debugf("while syncing hub dir: %s", err) @@ -398,6 +396,7 @@ func (h *Hub) syncDir(dir string) ([]string, error) { cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s: %s", cpath, err) + continue } // explicit check for non existing directory, avoid spamming log.Debug diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats index 9d0f24ba4c9..f49d0e24bf6 100644 --- a/test/bats/20_hub_collections.bats +++ b/test/bats/20_hub_collections.bats @@ -79,7 +79,17 @@ teardown() { rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" - # XXX: check alphabetical order in human, json, raw + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli collections list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli collections list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli collections list -o json -a | jq -r '.collections[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" } @test "cscli collections list [collection]..." { @@ -120,7 +130,7 @@ teardown() { assert_output "2" } -@test "cscli collections install [collection]..." { +@test "cscli collections install" { rune -1 cscli collections install assert_stderr --partial 'requires at least 1 arg(s), only received 0' @@ -148,8 +158,7 @@ teardown() { assert_output --partial 'installed: true' } -@test "cscli collections install [collection]... (file location and download-only)" { - # simple install +@test "cscli collections install (file location and download-only)" { rune -0 cscli collections install crowdsecurity/linux --download-only rune -0 cscli collections inspect crowdsecurity/linux --no-metrics assert_output --partial 'crowdsecurity/linux' @@ -158,11 +167,34 @@ teardown() { assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml" rune -0 cscli collections install crowdsecurity/linux + rune -0 cscli collections inspect crowdsecurity/linux --no-metrics + assert_output --partial 'installed: true' assert_file_exists "$CONFIG_DIR/collections/linux.yaml" } +@test "cscli collections install --force (tainted)" { + rune -0 cscli collections install crowdsecurity/sshd + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + + rune -1 cscli collections install crowdsecurity/sshd + assert_stderr --partial "error while installing 'crowdsecurity/sshd': while enabling crowdsecurity/sshd: crowdsecurity/sshd is tainted, won't enable unless --force" + + rune -0 cscli collections install crowdsecurity/sshd --force + assert_stderr --partial "crowdsecurity/sshd: overwrite" + assert_stderr --partial "Enabled crowdsecurity/sshd" +} + +@test "cscli collections install --ignore (skip on errors)" { + rune -1 cscli collections install foo/bar crowdsecurity/sshd + assert_stderr --partial "can't find 'foo/bar' in collections" + refute_stderr --partial "Enabled collections: crowdsecurity/sshd" + + rune -0 cscli collections install foo/bar crowdsecurity/sshd --ignore + assert_stderr --partial "can't find 'foo/bar' in collections" + assert_stderr --partial "Enabled collections: crowdsecurity/sshd" +} -@test "cscli collections inspect [collection]..." { +@test "cscli collections inspect" { rune -1 cscli collections inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' # required for metrics @@ -223,7 +255,7 @@ teardown() { assert_output "0" } -@test "cscli collections remove [collection]..." { +@test "cscli collections remove" { rune -1 cscli collections remove assert_stderr --partial "specify at least one collection to remove or '--all'" rune -1 cscli collections remove blahblah/blahblah @@ -277,7 +309,7 @@ teardown() { assert_output "0" } -@test "cscli collections remove [collections]... --force" { +@test "cscli collections remove --force" { # remove a collections that belongs to a collection rune -0 cscli collections install crowdsecurity/linux rune -0 cscli collections remove crowdsecurity/sshd @@ -285,7 +317,7 @@ teardown() { assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection" } -@test "cscli collections upgrade [collection]..." { +@test "cscli collections upgrade" { rune -1 cscli collections upgrade assert_stderr --partial "specify at least one collection to upgrade or '--all'" rune -1 cscli collections upgrade blahblah/blahblah diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats index c17b63315f0..c780457b3e6 100644 --- a/test/bats/20_hub_parsers.bats +++ b/test/bats/20_hub_parsers.bats @@ -79,7 +79,17 @@ teardown() { rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" - # XXX: check alphabetical order in human, json, raw + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" } @test "cscli parsers list [parser]..." { @@ -120,7 +130,7 @@ teardown() { assert_output "3" } -@test "cscli parsers install [parser]..." { +@test "cscli parsers install" { rune -1 cscli parsers install assert_stderr --partial 'requires at least 1 arg(s), only received 0' @@ -148,8 +158,7 @@ teardown() { assert_output --partial 'installed: true' } -@test "cscli parsers install [parser]... (file location and download-only)" { - # simple install +@test "cscli parsers install (file location and download-only)" { rune -0 cscli parsers install crowdsecurity/whitelists --download-only rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics assert_output --partial 'crowdsecurity/whitelists' @@ -158,13 +167,34 @@ teardown() { assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'installed: true' assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" } -# XXX: test install with --force -# XXX: test install with --ignore +@test "cscli parsers install --force (tainted)" { + rune -0 cscli parsers install crowdsecurity/whitelists + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -1 cscli parsers install crowdsecurity/whitelists + assert_stderr --partial "error while installing 'crowdsecurity/whitelists': while enabling crowdsecurity/whitelists: crowdsecurity/whitelists is tainted, won't enable unless --force" + + rune -0 cscli parsers install crowdsecurity/whitelists --force + assert_stderr --partial "crowdsecurity/whitelists: overwrite" + assert_stderr --partial "Enabled crowdsecurity/whitelists" +} + +@test "cscli parsers install --ignore (skip on errors)" { + rune -1 cscli parsers install foo/bar crowdsecurity/whitelists + assert_stderr --partial "can't find 'foo/bar' in parsers" + refute_stderr --partial "Enabled parsers: crowdsecurity/whitelists" -@test "cscli parsers inspect [parser]..." { + rune -0 cscli parsers install foo/bar crowdsecurity/whitelists --ignore + assert_stderr --partial "can't find 'foo/bar' in parsers" + assert_stderr --partial "Enabled parsers: crowdsecurity/whitelists" +} + +@test "cscli parsers inspect" { rune -1 cscli parsers inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' # required for metrics @@ -195,8 +225,8 @@ teardown() { # one item, raw rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw assert_line 'type: parsers' - assert_line 'stage: s01-parse' assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'stage: s01-parse' assert_line 'author: crowdsecurity' assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' assert_line 'installed: false' @@ -227,7 +257,7 @@ teardown() { assert_output "0" } -@test "cscli parsers remove [parser]..." { +@test "cscli parsers remove" { rune -1 cscli parsers remove assert_stderr --partial "specify at least one parser to remove or '--all'" rune -1 cscli parsers remove blahblah/blahblah @@ -281,7 +311,7 @@ teardown() { assert_output "0" } -@test "cscli parsers remove [parser]... --force" { +@test "cscli parsers remove --force" { # remove a parser that belongs to a collection rune -0 cscli collections install crowdsecurity/sshd rune -0 cscli parsers remove crowdsecurity/sshd-logs @@ -289,7 +319,7 @@ teardown() { assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser" } -@test "cscli parsers upgrade [parser]..." { +@test "cscli parsers upgrade" { rune -1 cscli parsers upgrade assert_stderr --partial "specify at least one parser to upgrade or '--all'" rune -1 cscli parsers upgrade blahblah/blahblah diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats index 23c2ef20bec..55c384942cb 100644 --- a/test/bats/20_hub_postoverflows.bats +++ b/test/bats/20_hub_postoverflows.bats @@ -79,15 +79,24 @@ teardown() { rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" - # XXX: check alphabetical order in human, json, raw + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli postoverflows list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli postoverflows list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli postoverflows list -o json -a | jq -r '.postoverflows[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" } -@test "cscli postoverflows list [scenario]..." { +@test "cscli postoverflows list [postoverflow]..." { # non-existent rune -1 cscli postoverflows install foo/bar assert_stderr --partial "can't find 'foo/bar' in postoverflows" - # not installed rune -0 cscli postoverflows list crowdsecurity/rdns assert_output --regexp 'crowdsecurity/rdns.*disabled' @@ -121,7 +130,7 @@ teardown() { assert_output "3" } -@test "cscli postoverflows install [scenario]..." { +@test "cscli postoverflows install" { rune -1 cscli postoverflows install assert_stderr --partial 'requires at least 1 arg(s), only received 0' @@ -149,8 +158,7 @@ teardown() { assert_output --partial 'installed: true' } -@test "cscli postoverflows install [postoverflow]... (file location and download-only)" { - # simple install +@test "cscli postoverflows install (file location and download-only)" { rune -0 cscli postoverflows install crowdsecurity/rdns --download-only rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics assert_output --partial 'crowdsecurity/rdns' @@ -159,13 +167,34 @@ teardown() { assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'installed: true' assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" } -# XXX: test install with --force -# XXX: test install with --ignore +@test "cscli postoverflows install --force (tainted)" { + rune -0 cscli postoverflows install crowdsecurity/rdns + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + rune -1 cscli postoverflows install crowdsecurity/rdns + assert_stderr --partial "error while installing 'crowdsecurity/rdns': while enabling crowdsecurity/rdns: crowdsecurity/rdns is tainted, won't enable unless --force" -@test "cscli postoverflows inspect [scenario]..." { + rune -0 cscli postoverflows install crowdsecurity/rdns --force + assert_stderr --partial "crowdsecurity/rdns: overwrite" + assert_stderr --partial "Enabled crowdsecurity/rdns" +} + +@test "cscli postoverflow install --ignore (skip on errors)" { + rune -1 cscli postoverflows install foo/bar crowdsecurity/rdns + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + refute_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" + + rune -0 cscli postoverflows install foo/bar crowdsecurity/rdns --ignore + assert_stderr --partial "can't find 'foo/bar' in postoverflows" + assert_stderr --partial "Enabled postoverflows: crowdsecurity/rdns" +} + +@test "cscli postoverflows inspect" { rune -1 cscli postoverflows inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' # required for metrics @@ -196,8 +225,8 @@ teardown() { # one item, raw rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw assert_line 'type: postoverflows' - assert_line 'stage: s00-enrich' assert_line 'name: crowdsecurity/rdns' + assert_line 'stage: s00-enrich' assert_line 'author: crowdsecurity' assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' assert_line 'installed: false' @@ -228,7 +257,7 @@ teardown() { assert_output "0" } -@test "cscli postoverflows remove [postoverflow]..." { +@test "cscli postoverflows remove" { rune -1 cscli postoverflows remove assert_stderr --partial "specify at least one postoverflow to remove or '--all'" rune -1 cscli postoverflows remove blahblah/blahblah @@ -282,7 +311,7 @@ teardown() { assert_output "0" } -@test "cscli postoverflows remove [postoverflow]... --force" { +@test "cscli postoverflows remove --force" { # remove a postoverflow that belongs to a collection rune -0 cscli collections install crowdsecurity/auditd rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process @@ -290,7 +319,7 @@ teardown() { assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow" } -@test "cscli postoverflows upgrade [postoverflow]..." { +@test "cscli postoverflows upgrade" { rune -1 cscli postoverflows upgrade assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" rune -1 cscli postoverflows upgrade blahblah/blahblah diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats index 8e6d2172563..bf033c2f949 100644 --- a/test/bats/20_hub_scenarios.bats +++ b/test/bats/20_hub_scenarios.bats @@ -79,7 +79,17 @@ teardown() { rune -0 grep -vc 'name,status,version,description' <(output) assert_output "$expected" - # XXX: check alphabetical order in human, json, raw + # the list should be the same in all formats, and sorted (not case sensitive) + + list_raw=$(cscli scenarios list -o raw -a | tail -n +2 | cut -d, -f1) + list_human=$(cscli scenarios list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2) + list_json=$(cscli scenarios list -o json -a | jq -r '.scenarios[].name') + + rune -0 sort -f <<<"$list_raw" + assert_output "$list_raw" + + assert_equal "$list_raw" "$list_json" + assert_equal "$list_raw" "$list_human" } @test "cscli scenarios list [scenario]..." { @@ -120,7 +130,7 @@ teardown() { assert_output "3" } -@test "cscli scenarios install [scenario]..." { +@test "cscli scenarios install" { rune -1 cscli scenarios install assert_stderr --partial 'requires at least 1 arg(s), only received 0' @@ -148,7 +158,7 @@ teardown() { assert_output --partial 'installed: true' } -@test "cscli scenarios install [scenario]... (file location and download-only)" { +@test "cscli scenarios install (file location and download-only)" { # simple install rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics @@ -158,14 +168,34 @@ teardown() { assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'installed: true' assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" } -# XXX: test install with --force -# XXX: test install with --ignore +@test "cscli scenarios install --force (tainted)" { + rune -0 cscli scenarios install crowdsecurity/ssh-bf + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + + rune -1 cscli scenarios install crowdsecurity/ssh-bf + assert_stderr --partial "error while installing 'crowdsecurity/ssh-bf': while enabling crowdsecurity/ssh-bf: crowdsecurity/ssh-bf is tainted, won't enable unless --force" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf --force + assert_stderr --partial "crowdsecurity/ssh-bf: overwrite" + assert_stderr --partial "Enabled crowdsecurity/ssh-bf" +} + +@test "cscli scenarios install --ignore (skip on errors)" { + rune -1 cscli scenarios install foo/bar crowdsecurity/ssh-bf + assert_stderr --partial "can't find 'foo/bar' in scenarios" + refute_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" + rune -0 cscli scenarios install foo/bar crowdsecurity/ssh-bf --ignore + assert_stderr --partial "can't find 'foo/bar' in scenarios" + assert_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf" +} -@test "cscli scenarios inspect [scenario]..." { +@test "cscli scenarios inspect" { rune -1 cscli scenarios inspect assert_stderr --partial 'requires at least 1 arg(s), only received 0' # required for metrics @@ -226,7 +256,7 @@ teardown() { assert_output "0" } -@test "cscli scenarios remove [scenario]..." { +@test "cscli scenarios remove" { rune -1 cscli scenarios remove assert_stderr --partial "specify at least one scenario to remove or '--all'" rune -1 cscli scenarios remove blahblah/blahblah @@ -280,7 +310,7 @@ teardown() { assert_output "0" } -@test "cscli scenarios remove [scenario]... --force" { +@test "cscli scenarios remove --force" { # remove a scenario that belongs to a collection rune -0 cscli collections install crowdsecurity/sshd rune -0 cscli scenarios remove crowdsecurity/ssh-bf @@ -288,7 +318,7 @@ teardown() { assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario" } -@test "cscli scenarios upgrade [scenario]..." { +@test "cscli scenarios upgrade" { rune -1 cscli scenarios upgrade assert_stderr --partial "specify at least one scenario to upgrade or '--all'" rune -1 cscli scenarios upgrade blahblah/blahblah From c8af58d1bf35d01552e1d5046a0e56f9e2892e3e Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 15 Nov 2023 17:46:31 +0100 Subject: [PATCH 174/263] ensure we're sending lapi/capi alert if the request matched some inband rules --- pkg/acquisition/modules/waap/utils.go | 12 ++++++++++-- pkg/acquisition/modules/waap/waap_runner.go | 4 ++-- pkg/types/waap_event.go | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index da25daa6f89..e302f868212 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -16,7 +16,11 @@ import ( log "github.com/sirupsen/logrus" ) -func WaapEventGeneration(inEvt types.Event) (types.Event, error) { +func WaapEventGeneration(inEvt types.Event) (*types.Event, error) { + //if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI + if !inEvt.Waap.HasInBandMatches { + return nil, nil + } evt := types.Event{} evt.Type = types.WAAP evt.Process = true @@ -62,7 +66,7 @@ func WaapEventGeneration(inEvt types.Event) (types.Event, error) { evt.Overflow.APIAlerts = []models.Alert{alert} evt.Overflow.Alert = &alert - return evt, nil + return &evt, nil } func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { @@ -193,7 +197,11 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest kind := "outofband" if req.IsInBand { kind = "inband" + evt.Waap.HasInBandMatches = true + } else { + evt.Waap.HasOutBandMatches = true } + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() spew.Dump(waf.WaapRulesDetails) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index fa80141ed86..a8f83d41e4c 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -253,8 +253,8 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { waapOvlfw, err := WaapEventGeneration(evt) if err != nil { r.logger.Errorf("unable to generate waap event : %s", err) - } else { - r.outChan <- waapOvlfw + } else if waapOvlfw != nil { + r.outChan <- *waapOvlfw } } } diff --git a/pkg/types/waap_event.go b/pkg/types/waap_event.go index 9b474be4ddd..c0c89408e6f 100644 --- a/pkg/types/waap_event.go +++ b/pkg/types/waap_event.go @@ -20,6 +20,7 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 type MatchedRules []map[string]interface{} type WaapEvent struct { + HasInBandMatches, HasOutBandMatches bool MatchedRules Vars map[string]string } From d9b0d440bfad716221ef81eeec886be532491e0c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Nov 2023 11:09:49 +0100 Subject: [PATCH 175/263] Refact cwhub (#2596) * unused param * (slightly) simpler ListItems() -> listItems() * listItems(): always showHeader, deduce showType ref. https://github.com/crowdsecurity/crowdsec/issues/1068 * simplify Item.disable() also, .tainted and .installed do not need a default since they are always in the json output now * Drop unused parameters --- cmd/crowdsec-cli/hub.go | 11 +++++++- cmd/crowdsec-cli/itemcommands.go | 12 +++++++-- cmd/crowdsec-cli/items.go | 38 +++++++++------------------ cmd/crowdsec-cli/support.go | 11 +++++++- cmd/crowdsec-cli/utils_table.go | 2 +- pkg/cwhub/enable.go | 20 ++++---------- test/bats/20_hub.bats | 2 +- test/bats/20_hub_collections_dep.bats | 12 ++++++--- 8 files changed, 59 insertions(+), 49 deletions(-) diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 0b61cec4b97..ad3110011c6 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -54,7 +54,16 @@ func runHubList(cmd *cobra.Command, args []string) error { log.Info(line) } - err = ListItems(hub, color.Output, cwhub.ItemTypes, nil, true, false, all) + items := make(map[string][]*cwhub.Item) + + for _, itemType := range cwhub.ItemTypes { + items[itemType], err = selectItems(hub, itemType, nil, !all) + if err != nil { + return err + } + } + + err = listItems(color.Output, cwhub.ItemTypes, items) if err != nil { return err } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index f93e3b0b9b2..de7aab68ac1 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/coalesce" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) type cmdHelp struct { @@ -508,7 +509,7 @@ func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) if item == nil { return fmt.Errorf("can't find '%s' in %s", name, it.name) } - if err = InspectItem(hub, item, !noMetrics); err != nil { + if err = InspectItem(item, !noMetrics); err != nil { return err } } @@ -556,7 +557,14 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err return err } - if err = ListItems(hub, color.Output, []string{it.name}, args, false, true, all); err != nil { + items := make(map[string][]*cwhub.Item) + + items[it.name], err = selectItems(hub, it.name, args, !all) + if err != nil { + return err + } + + if err = listItems(color.Output, []string{it.name}, items); err != nil { return err } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index 8104b1ed48d..aeefd426303 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -53,23 +53,11 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b return items, nil } -// XXX: too complex, should be two functions (itemtypes array and args are not used together) -func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { - items := make(map[string][]*cwhub.Item) - - for _, itemType := range itemTypes { - selected, err := selectItems(hub, itemType, args, !all) - if err != nil { - return err - } - - items[itemType] = selected - } - +func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error { switch csConfig.Cscli.Output { case "human": for _, itemType := range itemTypes { - listHubItemTable(hub, out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) + listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType]) } case "json": type itemHubStatus struct { @@ -98,23 +86,23 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, } } } + x, err := json.MarshalIndent(hubStatus, "", " ") if err != nil { return fmt.Errorf("failed to unmarshal: %w", err) } + out.Write(x) case "raw": csvwriter := csv.NewWriter(out) - if showHeader { - header := []string{"name", "status", "version", "description"} - if showType { - header = append(header, "type") - } - err := csvwriter.Write(header) - if err != nil { - return fmt.Errorf("failed to write header: %s", err) - } + header := []string{"name", "status", "version", "description"} + if len(itemTypes) > 1 { + header = append(header, "type") + } + + if err := csvwriter.Write(header); err != nil { + return fmt.Errorf("failed to write header: %s", err) } for _, itemType := range itemTypes { @@ -126,7 +114,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, item.LocalVersion, item.Description, } - if showType { + if len(itemTypes) > 1 { row = append(row, itemType) } if err := csvwriter.Write(row); err != nil { @@ -142,7 +130,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, return nil } -func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error { +func InspectItem(item *cwhub.Item, showMetrics bool) error { switch csConfig.Cscli.Output { case "human", "raw": enc := yaml.NewEncoder(os.Stdout) diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 76043d2ee72..7841e1fc469 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -129,9 +129,18 @@ func collectOSInfo() ([]byte, error) { } func collectHubItems(hub *cwhub.Hub, itemType string) []byte { + var err error + out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - if err := ListItems(hub, out, []string{itemType}, []string{}, false, true, false); err != nil { + + items := make(map[string][]*cwhub.Item) + + if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } + + if err := listItems(out, []string{itemType}, items); err != nil { log.Warnf("could not collect %s list: %s", itemType, err) } return out.Bytes() diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 67a7a4ec547..b4f86718e40 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -11,7 +11,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, items []*cwhub.Item) { +func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index abe1d5e21b3..2f3bdb00b8e 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -110,24 +110,14 @@ func (i *Item) disable(purge bool, force bool) error { return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name) } - // disable sub-items if any - it's a collection for _, sub := range i.SubItems() { - // check if the item doesn't belong to another collection before removing it - removeSub := true - - for _, collection := range sub.BelongsToCollections { - if collection != i.Name { - removeSub = false - break - } + if len(sub.BelongsToCollections) > 1 { + log.Infof("%s was not removed because it belongs to another collection", sub.Name) + continue } - if removeSub { - if err := sub.disable(purge, force); err != nil { - return fmt.Errorf("while disabling %s: %w", sub.Name, err) - } - } else { - log.Infof("%s was not removed because it belongs to another collection", sub.Name) + if err := sub.disable(purge, force); err != nil { + return fmt.Errorf("while disabling %s: %w", sub.Name, err) } } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index c2dd2795964..c1fd4e99f4e 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -38,7 +38,7 @@ teardown() { rune -0 cscli hub list -o json assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw - refute_output + assert_output 'name,status,version,description,type' # some items rune -0 cscli parsers install crowdsecurity/whitelists diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index 3e8a1111042..b3dc8077586 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -62,7 +62,7 @@ teardown() { # and now smb is tainted! rune -0 cscli collections inspect crowdsecurity/smb -o json - rune -0 jq -e '.tainted//false==true' <(output) + rune -0 jq -e '.tainted==true' <(output) rune -0 cscli collections remove crowdsecurity/smb --force # empty @@ -74,12 +74,18 @@ teardown() { # taint on sshd means smb is tainted as well rune -0 cscli collections inspect crowdsecurity/smb -o json - jq -e '.tainted//false==false' <(output) + rune -0 jq -e '.tainted==false' <(output) echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" rune -0 cscli collections inspect crowdsecurity/smb -o json - jq -e '.tainted//false==true' <(output) + rune -0 jq -e '.tainted==true' <(output) # now we can't remove smb without --force rune -1 cscli collections remove crowdsecurity/smb assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite" + + rune -0 cscli collections install crowdsecurity/wireguard baudneo/gotify + rune -0 cscli collections remove crowdsecurity/wireguard + assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it belongs to another collection" + rune -0 cscli collections inspect crowdsecurity/wireguard -o json + rune -0 jq -e '.installed==false' <(output) } From 65473d4e05112bf3e0a1852b3eba717d2257a8b0 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:05:55 +0100 Subject: [PATCH 176/263] Refact cwhub: simplify enable/disable/download (#2597) * Extract methods createInstallLink(), removeInstallLink(), simplify - the result of filepath.Join is already Cleaned - no need to log the creation of parentDir - filepath.Abs() only returns error if the current working directory has been removed * Extract method Item.fetch() * Replace Create() + Write() -> WriteFile() --- pkg/cwhub/cwhub_test.go | 7 +- pkg/cwhub/enable.go | 164 +++++++++++++++++++++------------------- pkg/cwhub/helpers.go | 84 ++++++++++---------- pkg/cwhub/hub_test.go | 2 +- pkg/cwhub/items.go | 2 +- pkg/cwhub/remote.go | 13 +--- 6 files changed, 134 insertions(+), 138 deletions(-) diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index fd95dd2eba0..3a260cb9a75 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -48,14 +48,9 @@ func testHub(t *testing.T, update bool) *Hub { err = os.MkdirAll(local.InstallDataDir, 0o700) require.NoError(t, err) - index, err := os.Create(local.HubIndexFile) + err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) require.NoError(t, err) - _, err = index.WriteString(`{}`) - require.NoError(t, err) - - index.Close() - t.Cleanup(func() { os.RemoveAll(tmpDir) }) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 2f3bdb00b8e..9887558f60c 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -10,12 +10,42 @@ import ( log "github.com/sirupsen/logrus" ) -// enable creates a symlink between actual config file at hub.HubDir and hub.ConfigDir -// Handles collections recursively -func (i *Item) enable() error { - parentDir := filepath.Clean(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/") +// installLink returns the location of the symlink to the actual config file (eg. /etc/crowdsec/collections/xyz.yaml) +func (i *Item) installLink() string { + return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName) +} + +// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir +func (i *Item) createInstallLink() error { + dest, err := filepath.Abs(i.installLink()) + if err != nil { + return err + } + + destDir := filepath.Dir(dest) + if err = os.MkdirAll(destDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating %s: %w", destDir, err) + } + + if _, err = os.Lstat(dest); !os.IsNotExist(err) { + log.Infof("%s already exists.", dest) + return nil + } + + src, err := filepath.Abs(filepath.Join(i.hub.local.HubDir, i.RemotePath)) + if err != nil { + return err + } + + if err = os.Symlink(src, dest); err != nil { + return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err) + } + + return nil +} - // create directories if needed +// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items +func (i *Item) enable() error { if i.Installed { if i.Tainted { return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) @@ -32,40 +62,14 @@ func (i *Item) enable() error { } } - if _, err := os.Stat(parentDir); os.IsNotExist(err) { - log.Infof("%s doesn't exist, create", parentDir) - - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating directory: %w", err) - } - } - - // install sub-items if any for _, sub := range i.SubItems() { if err := sub.enable(); err != nil { return fmt.Errorf("while installing %s: %w", sub.Name, err) } } - // check if file already exists where it should in configdir (eg /etc/crowdsec/collections/) - if _, err := os.Lstat(parentDir + "/" + i.FileName); !os.IsNotExist(err) { - log.Infof("%s already exists.", parentDir+"/"+i.FileName) - return nil - } - - // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) - if err != nil { - return fmt.Errorf("while getting source path: %w", err) - } - - dstPath, err := filepath.Abs(parentDir + "/" + i.FileName) - if err != nil { - return fmt.Errorf("while getting destination path: %w", err) - } - - if err = os.Symlink(srcPath, dstPath); err != nil { - return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err) + if err := i.createInstallLink(); err != nil { + return err } log.Infof("Enabled %s: %s", i.Type, i.Name) @@ -81,12 +85,11 @@ func (i *Item) purge() error { return nil } - itempath := i.hub.local.HubDir + "/" + i.RemotePath + src := filepath.Join(i.hub.local.HubDir, i.RemotePath) - // disable hub file - if err := os.Remove(itempath); err != nil { + if err := os.Remove(src); err != nil { if os.IsNotExist(err) { - log.Debugf("%s doesn't exist, no need to remove", itempath) + log.Debugf("%s doesn't exist, no need to remove", src) return nil } @@ -94,7 +97,48 @@ func (i *Item) purge() error { } i.Downloaded = false - log.Infof("Removed source file [%s]: %s", i.Name, itempath) + log.Infof("Removed source file [%s]: %s", i.Name, src) + + return nil +} + +func (i *Item) removeInstallLink() error { + syml, err := filepath.Abs(i.installLink()) + if err != nil { + return err + } + + stat, err := os.Lstat(syml) + if err != nil { + return err + } + + // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... + if stat.Mode()&os.ModeSymlink == 0 { + log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + hubpath, err := os.Readlink(syml) + if err != nil { + return fmt.Errorf("while reading symlink: %w", err) + } + + src, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) + if err != nil { + return err + } + + if hubpath != src { + log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src) + return fmt.Errorf("%s isn't managed by hub", i.Name) + } + + if err := os.Remove(syml); err != nil { + return fmt.Errorf("while removing symlink: %w", err) + } + + log.Infof("Removed symlink [%s]: %s", i.Name, syml) return nil } @@ -111,6 +155,7 @@ func (i *Item) disable(purge bool, force bool) error { } for _, sub := range i.SubItems() { + // TODO XXX: if the other collection(s) are direct or indirect dependencies of the current one, it's good to go if len(sub.BelongsToCollections) > 1 { log.Infof("%s was not removed because it belongs to another collection", sub.Name) continue @@ -126,48 +171,13 @@ func (i *Item) disable(purge bool, force bool) error { return nil } - syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName) - if err != nil { - return err - } - - stat, err := os.Lstat(syml) + err := i.removeInstallLink() if os.IsNotExist(err) { - // we only accept to "delete" non existing items if it's a forced purge if !purge && !force { - return fmt.Errorf("can't delete %s: %s doesn't exist", i.Name, syml) + return fmt.Errorf("can't disable %s: %s doesn't exist", i.Name, i.installLink()) } - } else { - // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... - if stat.Mode()&os.ModeSymlink == 0 { - log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) - return fmt.Errorf("%s isn't managed by hub", i.Name) - } - - hubpath, err := os.Readlink(syml) - if err != nil { - return fmt.Errorf("while reading symlink: %w", err) - } - - absPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) - if err != nil { - return fmt.Errorf("while abs path: %w", err) - } - - if hubpath != absPath { - log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, absPath) - return fmt.Errorf("%s isn't managed by hub", i.Name) - } - - if err := os.Remove(syml); err != nil { - if os.IsNotExist(err) { - log.Debugf("%s doesn't exist, no need to remove", syml) - return nil - } - return fmt.Errorf("while removing symlink: %w", err) - } - - log.Infof("Removed symlink [%s]: %s", i.Name, syml) + } else if err != nil { + return err } i.Installed = false diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 2427a8fcb07..ba14c07fbb7 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -161,45 +161,31 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { return nil } -func (i *Item) download(overwrite bool) error { +// fetch downloads the item from the hub, verifies the hash and returns the body +func (i *Item) fetch() ([]byte, error) { url, err := i.hub.remote.urlTo(i.RemotePath) if err != nil { - return fmt.Errorf("failed to build hub item request: %w", err) - } - - tdir := i.hub.local.HubDir - - // if user didn't --force, don't overwrite local, tainted, up-to-date files - if !overwrite { - if i.Tainted { - log.Debugf("%s: tainted, not updated", i.Name) - return nil - } - - if i.UpToDate { - // We still have to check if data files are present - log.Debugf("%s: up-to-date, not updated", i.Name) - } + return nil, fmt.Errorf("failed to build hub item request: %w", err) } resp, err := hubClient.Get(url) if err != nil { - return fmt.Errorf("while downloading %s: %w", url, err) + return nil, fmt.Errorf("while downloading %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) + return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("while downloading %s: %w", url, err) + return nil, fmt.Errorf("while downloading %s: %w", url, err) } hash := sha256.New() if _, err = hash.Write(body); err != nil { - return fmt.Errorf("while hashing %s: %w", i.Name, err) + return nil, fmt.Errorf("while hashing %s: %w", i.Name, err) } meow := hex.EncodeToString(hash.Sum(nil)) @@ -207,31 +193,50 @@ func (i *Item) download(overwrite bool) error { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest) - return fmt.Errorf("invalid download hash for %s", i.Name) + return nil, fmt.Errorf("invalid download hash for %s", i.Name) + } + + return body, nil +} + +// download downloads the item from the hub and writes it to the hub directory +func (i *Item) download(overwrite bool) error { + // if user didn't --force, don't overwrite local, tainted, up-to-date files + if !overwrite { + if i.Tainted { + log.Debugf("%s: tainted, not updated", i.Name) + return nil + } + + if i.UpToDate { + // We still have to check if data files are present + log.Debugf("%s: up-to-date, not updated", i.Name) + } + } + + body, err := i.fetch() + if err != nil { + return err } + tdir := i.hub.local.HubDir + //all good, install - //check if parent dir exists - tmpdirs := strings.Split(tdir+"/"+i.RemotePath, "/") - parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/") - // ensure that target file is within target dir - finalPath, err := filepath.Abs(tdir + "/" + i.RemotePath) + finalPath, err := filepath.Abs(filepath.Join(tdir, i.RemotePath)) if err != nil { - return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+i.RemotePath, err) + return err } + // ensure that target file is within target dir if !strings.HasPrefix(finalPath, tdir) { return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir) } - // check dir - if _, err = os.Stat(parentDir); os.IsNotExist(err) { - log.Debugf("%s doesn't exist, create", parentDir) + parentDir := filepath.Dir(finalPath) - if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating parent directories: %w", err) - } + if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { + return fmt.Errorf("while creating %s: %w", parentDir, err) } // check actual file @@ -242,15 +247,8 @@ func (i *Item) download(overwrite bool) error { log.Infof("%s: OK", i.Name) } - f, err := os.Create(tdir + "/" + i.RemotePath) - if err != nil { - return fmt.Errorf("while opening file: %w", err) - } - defer f.Close() - - _, err = f.Write(body) - if err != nil { - return fmt.Errorf("while writing file: %w", err) + if err = os.WriteFile(finalPath, body, 0o644); err != nil { + return fmt.Errorf("while writing %s: %w", finalPath, err) } i.Downloaded = true diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 1bba5e90a60..56e2bf376a7 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -69,5 +69,5 @@ func TestDownloadIndex(t *testing.T) { } err = hub.remote.downloadIndex("/does/not/exist/index.json") - cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") + cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:") } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 8e3063a9daa..37e853d3aee 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -250,7 +250,7 @@ func (i *Item) versionStatus() int { } // validPath returns true if the (relative) path is allowed for the item -// dirNmae: the directory name (ie. crowdsecurity) +// dirNname: the directory name (ie. crowdsecurity) // fileName: the filename (ie. apache2-logs.yaml) func (i *Item) validPath(dirName, fileName string) bool { return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index ad97a4efbb5..c3855d5e02d 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -70,18 +70,11 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error { return nil } - file, err := os.Create(localPath) - if err != nil { - return fmt.Errorf("while opening hub index file: %w", err) - } - defer file.Close() - - wsize, err := file.Write(body) - if err != nil { - return fmt.Errorf("while writing hub index file: %w", err) + if err = os.WriteFile(localPath, body, 0o644); err != nil { + return fmt.Errorf("failed to write hub index: %w", err) } - log.Infof("Wrote index to %s, %d bytes", localPath, wsize) + log.Infof("Wrote index to %s, %d bytes", localPath, len(body)) return nil } From 56ad2bbf9892247f2ea227f188ee9255c0ab6ed5 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:00:51 +0100 Subject: [PATCH 177/263] Refact cwhub: item removal with shared dependencies (#2598) * Iterate over sub-items in Remove(), not in disable() -- fix shared dependency issue * Increase hub download timeout to 2 minutes --- pkg/cwhub/cwhub.go | 2 +- pkg/cwhub/enable.go | 27 +---------- pkg/cwhub/helpers.go | 65 +++++++++++++++++++++++++-- pkg/cwhub/items.go | 15 +++++++ pkg/cwhub/sync.go | 2 + test/bats/20_hub_collections_dep.bats | 27 ++++++++++- test/bats/20_hub_items.bats | 2 +- 7 files changed, 107 insertions(+), 33 deletions(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 74d533c74f8..c7e17bd62f3 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -10,5 +10,5 @@ import ( ) var hubClient = &http.Client{ - Timeout: 20 * time.Second, + Timeout: 120 * time.Second, } diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 9887558f60c..49946c8693c 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -146,35 +146,10 @@ func (i *Item) removeInstallLink() error { // disable removes the symlink to the downloaded content, also removes the content if purge is true func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not - if i.IsLocal() { - return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) - } - - if i.Tainted && !force { - return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name) - } - - for _, sub := range i.SubItems() { - // TODO XXX: if the other collection(s) are direct or indirect dependencies of the current one, it's good to go - if len(sub.BelongsToCollections) > 1 { - log.Infof("%s was not removed because it belongs to another collection", sub.Name) - continue - } - - if err := sub.disable(purge, force); err != nil { - return fmt.Errorf("while disabling %s: %w", sub.Name, err) - } - } - - if !i.Installed && !purge { - log.Infof("removing %s: not installed -- no need to remove", i.Name) - return nil - } - err := i.removeInstallLink() if os.IsNotExist(err) { if !purge && !force { - return fmt.Errorf("can't disable %s: %s doesn't exist", i.Name, i.installLink()) + return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLink()) } } else if err != nil { return err diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index ba14c07fbb7..6d106f813c9 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -17,6 +17,7 @@ import ( "github.com/enescakir/emoji" log "github.com/sirupsen/logrus" + "slices" ) // Install installs the item from the hub, downloading it if needed @@ -51,12 +52,70 @@ func (i *Item) Install(force bool, downloadOnly bool) error { return nil } +// allDependencies return a list of all dependencies and sub-dependencies of the item +func (i *Item) allDependencies() []*Item { + var deps []*Item + + for _, dep := range i.SubItems() { + if dep == i { + log.Errorf("circular dependency detected: %s depends on %s", dep.Name, i.Name) + continue + } + + deps = append(deps, dep.allDependencies()...) + } + + return append(deps, i) +} + // Remove disables the item, optionally removing the downloaded content -func (i *Item) Remove(purge bool, forceAction bool) (bool, error) { +func (i *Item) Remove(purge bool, force bool) (bool, error) { + if i.IsLocal() { + return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) + } + + if i.Tainted && !force { + return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) + } + + if !i.Installed && !purge { + log.Infof("removing %s: not installed -- no need to remove", i.Name) + return false, nil + } + removed := false - if err := i.disable(purge, forceAction); err != nil { - return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) + allDeps := i.allDependencies() + + for _, sub := range i.SubItems() { + if !sub.Installed { + continue + } + + // if the other collection(s) are direct or indirect dependencies of the current one, it's good to go + // log parent collections + for _, subParent := range sub.parentCollections() { + if subParent == i { + continue + } + + if !slices.Contains(allDeps, subParent) { + log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name) + continue + } + } + + subRemoved, err := sub.Remove(purge, force) + if err != nil { + return false, fmt.Errorf("unable to disable %s: %w", i.Name, err) + } + + removed = removed || subRemoved + } + + err := i.disable(purge, force) + if err != nil { + return false, fmt.Errorf("while removing %s: %w", i.Name, err) } // XXX: should take the value from disable() diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 37e853d3aee..2ed593e2093 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -186,6 +186,21 @@ func (i *Item) logMissingSubItems() { } } +func (i *Item) parentCollections() []*Item { + ret := make([]*Item, 0) + + for _, parentName := range i.BelongsToCollections { + parent := i.hub.GetItem(COLLECTIONS, parentName) + if parent == nil { + continue + } + + ret = append(ret, parent) + } + + return ret +} + // Status returns the status of the item as a string and an emoji // ie. "enabled,update-available" and emoji.Warning func (i *Item) Status() (string, emoji.Emoji) { diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 30336fa91c1..2d55e9e21d6 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -393,6 +393,8 @@ func (h *Hub) syncDir(dir string) ([]string, error) { // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { + // cpath: top-level item directory, either downloaded or installed items. + // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s: %s", cpath, err) diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index b3dc8077586..a44d8bc9c78 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -81,11 +81,34 @@ teardown() { # now we can't remove smb without --force rune -1 cscli collections remove crowdsecurity/smb - assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite" + assert_stderr --partial "crowdsecurity/smb is tainted, use '--force' to remove" +} +@test "cscli collections (dependencies II: the revenge)" { rune -0 cscli collections install crowdsecurity/wireguard baudneo/gotify rune -0 cscli collections remove crowdsecurity/wireguard - assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it belongs to another collection" + assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it also belongs to baudneo/gotify" rune -0 cscli collections inspect crowdsecurity/wireguard -o json rune -0 jq -e '.installed==false' <(output) } + +@test "cscli collections (dependencies III: origins)" { + # it is perfectly fine to remove an item belonging to a collection that we are removing anyway + + # inject a dependency: sshd requires the syslog-logs parsers, but linux does too + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{parsers:["crowdsecurity/syslog-logs"]}}}') + echo "$hub_dep" >"$INDEX_PATH" + + # verify that installing sshd brings syslog-logs + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json + rune -0 jq -e '.installed==true' <(output) + + rune -0 cscli collections install crowdsecurity/linux + + # removing linux should remove syslog-logs even though sshd depends on it + rune -0 cscli collections remove crowdsecurity/linux + refute_stderr --partial "crowdsecurity/syslog-logs was not removed" + rune -0 cscli parsers list -o json + rune -0 jq -e '.parsers | length == 0' <(output) +} diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 0ac3d1c8d3d..ca40beea090 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -80,7 +80,7 @@ teardown() { rune -0 rm "$(output)" rune -0 cscli parsers remove crowdsecurity/syslog-logs --debug - assert_stderr --partial "Removed crowdsecurity/syslog-logs" + assert_stderr --partial "removing crowdsecurity/syslog-logs: not installed -- no need to remove" rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json rune -0 jq -r '.path' <(output) From 9db48e211065bf0fea598159cd12663ea05dbb96 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 16 Nov 2023 17:17:33 +0100 Subject: [PATCH 178/263] fix collections install/inspect with waap-{rules,configs} --- pkg/cwhub/items.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index f2558e2df14..facf53dfb5d 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -75,8 +75,8 @@ type Item struct { PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` - WaapConfigs []string `json:"waap_configs,omitempty" yaml:"waap_configs,omitempty"` - WaapRules []string `json:"waap_rules,omitempty" yaml:"waap_rules,omitempty"` + WaapConfigs []string `json:"waap-configs,omitempty" yaml:"waap-configs,omitempty"` + WaapRules []string `json:"waap-rules,omitempty" yaml:"waap-rules,omitempty"` } func (i *Item) HasSubItems() bool { From 9864d2c4594bb42018b0dbd9ba52380dceb9c5db Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Thu, 16 Nov 2023 18:19:32 +0100 Subject: [PATCH 179/263] Add authentication between bouncers and waf --- pkg/acquisition/modules/waap/waap.go | 102 ++++++++++++++++++++++++--- pkg/csconfig/config.go | 8 +++ pkg/waf/request.go | 9 +-- 3 files changed, 105 insertions(+), 14 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 92363b2e3d9..cf9fd2ad661 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -5,6 +5,10 @@ import ( "encoding/json" "fmt" "net/http" + "sync" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -23,16 +27,21 @@ const ( OutOfBand = "outofband" ) +var ( + DefaultAuthCacheDuration = (1 * time.Minute) +) + // configuration structure of the acquis for the Waap type WaapSourceConfig struct { - ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` - CertFilePath string `yaml:"cert_file"` - KeyFilePath string `yaml:"key_file"` - Path string `yaml:"path"` - Routines int `yaml:"routines"` - WaapConfig string `yaml:"waap_config"` - WaapConfigPath string `yaml:"waap_config_path"` + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` + CertFilePath string `yaml:"cert_file"` + KeyFilePath string `yaml:"key_file"` + Path string `yaml:"path"` + Routines int `yaml:"routines"` + WaapConfig string `yaml:"waap_config"` + WaapConfigPath string `yaml:"waap_config_path"` + AuthCacheDuration *time.Duration `yaml:"auth_cache_duration"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -46,12 +55,38 @@ type WaapSource struct { outChan chan types.Event InChan chan waf.ParsedRequest WaapRuntime *waf.WaapRuntimeConfig - WaapConfigs map[string]waf.WaapConfig - + lapiURL string + AuthCache AuthCache WaapRunners []WaapRunner //one for each go-routine } +// Struct to handle cache of authentication +type AuthCache struct { + APIKeys map[string]time.Time + mu sync.RWMutex +} + +func NewAuthCache() AuthCache { + return AuthCache{ + APIKeys: make(map[string]time.Time, 0), + mu: sync.RWMutex{}, + } +} + +func (ac *AuthCache) Set(apiKey string, expiration time.Time) { + ac.mu.Lock() + ac.APIKeys[apiKey] = expiration + ac.mu.Unlock() +} + +func (ac *AuthCache) Get(apiKey string) (time.Time, bool) { + ac.mu.RLock() + expiration, exists := ac.APIKeys[apiKey] + ac.mu.RUnlock() + return expiration, exists +} + // @tko + @sbl : we might want to get rid of that or improve it type BodyResponse struct { Action string `json:"action"` @@ -97,6 +132,11 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { if wc.config.WaapConfig == "" && wc.config.WaapConfigPath == "" { return fmt.Errorf("waap_config or waap_config_path must be set") } + + csConfig := csconfig.GetConfig() + wc.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL) + wc.AuthCache = NewAuthCache() + return nil } @@ -118,6 +158,11 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.logger.Tracef("WAF configuration: %+v", w.config) + if w.config.AuthCacheDuration == nil { + w.config.AuthCacheDuration = &DefaultAuthCacheDuration + w.logger.Infof("Cache duration for auth not set, using default: %v", *w.config.AuthCacheDuration) + } + w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) w.mux = http.NewServeMux() @@ -251,8 +296,44 @@ func (w *WaapSource) Dump() interface{} { return w } +func (w *WaapSource) IsAuth(apiKey string) bool { + client := &http.Client{} + req, err := http.NewRequest("HEAD", w.lapiURL, nil) + if err != nil { + fmt.Println("Error creating request:", err) + return false + } + req.Header.Add("X-Api-Key", apiKey) + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error performing request:", err) + return false + } + defer resp.Body.Close() + + return resp.StatusCode == http.StatusOK + +} + // should this be in the runner ? func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { + apiKey := r.Header.Get(waf.APIKeyHeaderName) + if apiKey == "" { + rw.WriteHeader(http.StatusUnauthorized) + return + } + expiration, exists := w.AuthCache.Get(apiKey) + // if the apiKey is not in cache or has expired, just recheck the auth + if !exists || time.Now().After(expiration) { + if !w.IsAuth(apiKey) { + rw.WriteHeader(http.StatusUnauthorized) + return + } + + // apiKey is valid, store it in cache + w.AuthCache.Set(apiKey, time.Now().Add(*w.config.AuthCacheDuration)) + } + // parse the request only once parsedRequest, err := waf.NewParsedRequestFromRequest(r) if err != nil { @@ -260,6 +341,7 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusInternalServerError) return } + w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index ce8f09bc09f..9e90c20cc88 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -21,6 +21,8 @@ var defaultConfigDir = "/etc/crowdsec" // defaultDataDir is the base path to all data files, to be overridden in the Makefile */ var defaultDataDir = "/var/lib/crowdsec/data/" +var globalConfig = Config{} + // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags type Config struct { //just a path to ourselves :p @@ -89,9 +91,15 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool return nil, "", err } + globalConfig = cfg + return &cfg, configData, nil } +func GetConfig() Config { + return globalConfig +} + // XXX: We must not have a different behavior with an empty vs a missing configuration file. // XXX: For this reason, all defaults have to come from NewConfig(). The following function should // XXX: be replaced diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 5a7cc04a3a2..f7b7a11b737 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -11,10 +11,11 @@ import ( ) const ( - URIHeaderName = "X-Crowdsec-Waf-Uri" - VerbHeaderName = "X-Crowdsec-Waf-Verb" - HostHeaderName = "X-Crowdsec-Waf-Host" - IPHeaderName = "X-Crowdsec-Waf-Ip" + URIHeaderName = "X-Crowdsec-Waf-Uri" + VerbHeaderName = "X-Crowdsec-Waf-Verb" + HostHeaderName = "X-Crowdsec-Waf-Host" + IPHeaderName = "X-Crowdsec-Waf-Ip" + APIKeyHeaderName = "X-Crowdsec-Waf-Api-Key" ) // type ResponseRequest struct { From d40e9fb7605f56dfb5beb651e0a0ca44921771fe Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 17 Nov 2023 13:45:35 +0100 Subject: [PATCH 180/263] do not use filepath.Match --- pkg/waf/waap_rules_collection.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 0b4caccdcea..6c28d21dd7c 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -6,6 +6,7 @@ import ( "strings" corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" log "github.com/sirupsen/logrus" @@ -52,14 +53,14 @@ func LoadCollection(pattern string) ([]WaapCollection, error) { for _, waapRule := range waapRules { - matched, err := filepath.Match(pattern, waapRule.Name) + matched, err := exprhelpers.Match(pattern, waapRule.Name) if err != nil { log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) continue } - if !matched { + if !matched.(bool) { continue } From 0e717cb55859ae227bd436b0d5c410b01a7f78cc Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 17 Nov 2023 13:47:05 +0100 Subject: [PATCH 181/263] up --- pkg/waf/waap_rules_collection.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 6c28d21dd7c..1b579a73532 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -53,14 +53,21 @@ func LoadCollection(pattern string) ([]WaapCollection, error) { for _, waapRule := range waapRules { - matched, err := exprhelpers.Match(pattern, waapRule.Name) + tmpMatch, err := exprhelpers.Match(pattern, waapRule.Name) if err != nil { log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) continue } - if !matched.(bool) { + matched, ok := tmpMatch.(bool) + + if !ok { + log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) + continue + } + + if !matched { continue } From 55491be528c54920e8d7ba4d95c6ae920ea14b4e Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 17 Nov 2023 15:14:15 +0100 Subject: [PATCH 182/263] typo --- cmd/crowdsec-cli/hubtest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 8b574c3ee86..d1fc3830c91 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -128,7 +128,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios configFileData := &hubtest.HubTestItemConfig{ Parsers: parsers, Scenarios: scenarios, - PostOVerflows: postoverflows, + PostOverflows: postoverflows, LogFile: logFileName, LogType: logType, IgnoreParsers: ignoreParsers, From 9af30e2a3da47e6ff1692713225ce0b0cd3db28c Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 17 Nov 2023 15:15:29 +0100 Subject: [PATCH 183/263] simplify a bit --- pkg/hubtest/hubtest.go | 54 ++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index ec1f6ee5efe..0574d16db1a 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -11,22 +11,26 @@ import ( ) type HubTest struct { - CrowdSecPath string - CscliPath string - HubPath string - HubTestPath string - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - HubIndex *cwhub.Hub - Tests []*HubTestItem + CrowdSecPath string + CscliPath string + HubPath string + HubTestPath string + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateWaapProfilePath string + HubIndex *cwhub.Hub + Tests []*HubTestItem } const ( - templateConfigFile = "template_config.yaml" - templateSimulationFile = "template_simulation.yaml" - templateProfileFile = "template_profiles.yaml" + templateConfigFile = "template_config.yaml" + templateSimulationFile = "template_simulation.yaml" + templateProfileFile = "template_profiles.yaml" + templateAcquisFile = "template_acquis.yaml" + templateWaapProfilePath = "template_waap-profile.yaml" ) func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { @@ -70,20 +74,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, return HubTest{}, fmt.Errorf("unable to load hub: %s", err) } - templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile) - templateProfilePath := filepath.Join(HubTestPath, templateProfileFile) - templateSimulationPath := filepath.Join(HubTestPath, templateSimulationFile) - return HubTest{ - CrowdSecPath: crowdsecPath, - CscliPath: cscliPath, - HubPath: hubPath, - HubTestPath: HubTestPath, - HubIndexFile: hubIndexFile, - TemplateConfigPath: templateConfigFilePath, - TemplateProfilePath: templateProfilePath, - TemplateSimulationPath: templateSimulationPath, - HubIndex: hub, + CrowdSecPath: crowdsecPath, + CscliPath: cscliPath, + HubPath: hubPath, + HubTestPath: HubTestPath, + HubIndexFile: hubIndexFile, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), + TemplateWaapProfilePath: filepath.Join(HubTestPath, templateWaapProfilePath), + TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), + HubIndex: hub, }, nil } From 6718d8276578de25a3f197feb7a4584e1750aecf Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 17 Nov 2023 15:37:12 +0100 Subject: [PATCH 184/263] allow testing of waap rules --- pkg/hubtest/hubtest_item.go | 339 +++++++++++++++++++++++++++++------- 1 file changed, 272 insertions(+), 67 deletions(-) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 82107c6fe3a..65c95a2bc15 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -1,11 +1,14 @@ package hubtest import ( + "errors" "fmt" + "net" "os" "os/exec" "path/filepath" "strings" + "time" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -16,14 +19,17 @@ import ( ) type HubTestItemConfig struct { - Parsers []string `yaml:"parsers"` - Scenarios []string `yaml:"scenarios"` - PostOVerflows []string `yaml:"postoverflows"` - LogFile string `yaml:"log_file"` - LogType string `yaml:"log_type"` - Labels map[string]string `yaml:"labels"` - IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + Parsers []string `yaml:"parsers"` + Scenarios []string `yaml:"scenarios"` + PostOverflows []string `yaml:"postoverflows"` + WaapRules []string `yaml:"waap-rules"` + NucleiTemplate string `yaml:"nuclei_template"` + ExpectedNucleiFailure bool `yaml:"expect_failure"` + LogFile string `yaml:"log_file"` + LogType string `yaml:"log_type"` + Labels map[string]string `yaml:"labels"` + IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser + OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } type HubTestItem struct { @@ -40,6 +46,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string + RuntimeAcquisFilePath string RuntimeHubConfig *csconfig.LocalHubCfg ResultsPath string @@ -47,13 +54,15 @@ type HubTestItem struct { ScenarioResultFile string BucketPourResultFile string - HubPath string - HubTestPath string - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - HubIndex *cwhub.Hub + HubPath string + HubTestPath string + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateWaapProfilePath string + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -115,6 +124,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { RuntimeConfigFilePath: filepath.Join(runtimeFolder, "config.yaml"), RuntimeProfileFilePath: filepath.Join(runtimeFolder, "profiles.yaml"), RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"), + RuntimeAcquisFilePath: filepath.Join(runtimeFolder, "acquis.yaml"), ResultsPath: resultPath, ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), @@ -125,17 +135,19 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { InstallDir: runtimeFolder, InstallDataDir: filepath.Join(runtimeFolder, "data"), }, - Config: configFileData, - HubPath: hubTest.HubPath, - HubTestPath: hubTest.HubTestPath, - HubIndexFile: hubTest.HubIndexFile, - TemplateConfigPath: hubTest.TemplateConfigPath, - TemplateProfilePath: hubTest.TemplateProfilePath, - TemplateSimulationPath: hubTest.TemplateSimulationPath, - HubIndex: hubTest.HubIndex, - ScenarioAssert: ScenarioAssert, - ParserAssert: ParserAssert, - CustomItemsLocation: []string{hubTest.HubPath, testPath}, + Config: configFileData, + HubPath: hubTest.HubPath, + HubTestPath: hubTest.HubTestPath, + HubIndexFile: hubTest.HubIndexFile, + TemplateConfigPath: hubTest.TemplateConfigPath, + TemplateProfilePath: hubTest.TemplateProfilePath, + TemplateSimulationPath: hubTest.TemplateSimulationPath, + TemplateAcquisPath: hubTest.TemplateAcquisPath, + TemplateWaapProfilePath: hubTest.TemplateWaapProfilePath, + HubIndex: hubTest.HubIndex, + ScenarioAssert: ScenarioAssert, + ParserAssert: ParserAssert, + CustomItemsLocation: []string{hubTest.HubPath, testPath}, }, nil } @@ -297,8 +309,81 @@ func (t *HubTestItem) InstallHub() error { } } + // install waaprules in runtime environment + for _, waaprule := range t.Config.WaapRules { + log.Infof("adding rule '%s'", waaprule) + if waaprule == "" { + continue + } + + if hubWaapRule, ok := t.HubIndex.Items[cwhub.WAAP_RULES][waaprule]; ok { + waapRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubWaapRule.RemotePath)) + if err != nil { + return fmt.Errorf("can't get absolute path of '%s': %s", waapRuleSource, err) + } + + waapRuleFilename := filepath.Base(waapRuleSource) + + // runtime/hub/waap-rules/author/waap-rule + hubDirWaapRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubWaapRule.RemotePath)) + + // runtime/waap-rules/ + waapRuleDirDest := fmt.Sprintf("%s/waap-rules/", t.RuntimePath) + + if err := os.MkdirAll(hubDirWaapRuleDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", hubDirWaapRuleDest, err) + } + + if err := os.MkdirAll(waapRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", waapRuleDirDest, err) + } + + // runtime/hub/waap-rules/crowdsecurity/rule.yaml + hubDirWaapRulePath := filepath.Join(waapRuleDirDest, waapRuleFilename) + if err := Copy(waapRuleSource, hubDirWaapRulePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %s", waapRuleSource, hubDirWaapRulePath, err) + } + + // runtime/waap-rules/rule.yaml + waapRulePath := filepath.Join(waapRuleDirDest, waapRuleFilename) + if err := os.Symlink(hubDirWaapRulePath, waapRulePath); err != nil { + if !os.IsExist(err) { + return fmt.Errorf("unable to symlink waap-rule '%s' to '%s': %s", hubDirWaapRulePath, waapRulePath, err) + } + } + } else { + customWaapRuleExist := false + for _, customPath := range t.CustomItemsLocation { + // we check if its a custom waap-rule + customWaapRulePath := filepath.Join(customPath, waaprule) + if _, err := os.Stat(customWaapRulePath); os.IsNotExist(err) { + continue + } + customWaapRulePathSplit := strings.Split(customWaapRulePath, "/") + customWappRuleName := customWaapRulePathSplit[len(customWaapRulePathSplit)-1] + + waapRuleDirDest := fmt.Sprintf("%s/waap-rules/", t.RuntimePath) + if err := os.MkdirAll(waapRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", waapRuleDirDest, err) + } + + // runtime/waap-rules/ + customWaapRuleDest := fmt.Sprintf("%s/waap-rules/%s", t.RuntimePath, customWappRuleName) + // if path to postoverflow exist, copy it + if err := Copy(customWaapRulePath, customWaapRuleDest); err != nil { + continue + } + customWaapRuleExist = true + break + } + if !customWaapRuleExist { + return fmt.Errorf("couldn't find custom waap-rule '%s' in the following location: %+v", waaprule, t.CustomItemsLocation) + } + } + } + // install postoverflows in runtime environment - for _, postoverflow := range t.Config.PostOVerflows { + for _, postoverflow := range t.Config.PostOverflows { if postoverflow == "" { continue } @@ -449,69 +534,104 @@ func (t *HubTestItem) Clean() error { return os.RemoveAll(t.RuntimePath) } -func (t *HubTestItem) Run() error { - t.Success = false - t.ErrorsList = make([]string, 0) +func (t *HubTestItem) RunWithNucleiTemplate() error { testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) } - currentDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("can't get current directory: %+v", err) + if err := os.Chdir(testPath); err != nil { + return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err) } - // create runtime folder - if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) - } + //machine add + cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} + cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) - // create runtime data folder - if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) + output, err := cscliRegisterCmd.CombinedOutput() + if err != nil { + if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { + fmt.Println(string(output)) + return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + } } - // create runtime hub folder - if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) - } + //hardcode bouncer key + cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "waaptests", "-k", "this_is_a_bad_password"} + cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) - if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { - return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) + output, err = cscliBouncerCmd.CombinedOutput() + if err != nil { + if !strings.Contains(string(output), "unable to create bouncer: bouncer waaptests already exists") { + fmt.Println(string(output)) + return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + } } - // create results folder - if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) - } + //start crowdsec service + cmdArgs = []string{"-c", t.RuntimeConfigFilePath} + crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...) - // copy template config file to runtime folder - if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) - } + crowdsecDaemon.Start() - // copy template profile file to runtime folder - if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) + //wait for the waap port to be available + + start := time.Now() + + for { + conn, err := net.Dial("tcp", "127.0.0.1:4241") + if err == nil { + log.Debugf("waap is up after %s", time.Since(start)) + conn.Close() + break + } + time.Sleep(500 * time.Millisecond) + if time.Since(start) > 10*time.Second { + log.Fatalf("took more than 10s for waap to be available, abort") + } } - // copy template simulation file to runtime folder - if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) + nucleiConfig := NucleiConfig{ + Path: "nuclei", + OutputDir: testPath, + CmdLineOptions: []string{"-ev", //allow variables from environment + "-nc", //no colors in output + "-dresp", //dump response + "-j", //json output + }, } - crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, "http://127.0.0.1:80/") - // copy template patterns folder to runtime folder - if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { - return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) + if t.Config.ExpectedNucleiFailure { + if err != nil && errors.Is(err, NucleiTemplateFail) { + log.Infof("WAAP test %s failed as expected", t.Name) + t.Success = true + } else { + log.Errorf("WAAP test %s failed: %s", t.Name, err) + } + } else { + if err == nil { + log.Infof("WAAP test %s succeeded", t.Name) + t.Success = true + } else { + log.Errorf("WAAP test %s failed: %s", t.Name, err) + } } + crowdsecDaemon.Process.Kill() + return nil +} - // install the hub in the runtime folder - if err = t.InstallHub(); err != nil { - return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) +func (t *HubTestItem) RunWithLogFile() error { + testPath := filepath.Join(t.HubTestPath, t.Name) + if _, err := os.Stat(testPath); os.IsNotExist(err) { + return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) + } + + currentDir, err := os.Getwd() //xx + if err != nil { + return fmt.Errorf("can't get current directory: %+v", err) } logFile := t.Config.LogFile @@ -650,3 +770,88 @@ func (t *HubTestItem) Run() error { return nil } + +func (t *HubTestItem) Run() error { + var err error + t.Success = false + t.ErrorsList = make([]string, 0) + + // create runtime folder + if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) + } + + // create runtime data folder + if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) + } + + // create runtime hub folder + if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) + } + + if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { + return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) + } + + // create results folder + if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) + } + + // copy template config file to runtime folder + if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) + } + + // copy template profile file to runtime folder + if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) + } + + // copy template simulation file to runtime folder + if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) + } + + crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + + // copy template patterns folder to runtime folder + if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { + return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) + } + + // create the waap-configs dir + if err = os.MkdirAll(filepath.Join(t.RuntimePath, "waap-configs"), os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) + } + + // copy the waap-config file and acquis *only* if nuclei template is set -> it means we're testing waap + if t.Config.NucleiTemplate != "" { + // copy template acquis file to runtime folder + log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath) + if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err) + } + + log.Infof("copying %s to %s", t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml")) + // copy template waap-config file to runtime folder + if err = Copy(t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml")); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml"), err) + } + } + + // install the hub in the runtime folder + if err = t.InstallHub(); err != nil { + return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) + } + + if t.Config.LogFile != "" { + return t.RunWithLogFile() + } else if t.Config.NucleiTemplate != "" { + return t.RunWithNucleiTemplate() + } else { + return fmt.Errorf("log file or nuclei template must be set in '%s'", t.Name) + } +} From 017331ca7fd45ca962e2442132ef3366bcdc5313 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 17 Nov 2023 15:37:32 +0100 Subject: [PATCH 185/263] nuclei runner --- pkg/hubtest/nucleirunner.go | 61 +++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 pkg/hubtest/nucleirunner.go diff --git a/pkg/hubtest/nucleirunner.go b/pkg/hubtest/nucleirunner.go new file mode 100644 index 00000000000..048070a216f --- /dev/null +++ b/pkg/hubtest/nucleirunner.go @@ -0,0 +1,61 @@ +package hubtest + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +type NucleiConfig struct { + Path string `yaml:"nuclei_path"` + OutputDir string `yaml:"output_dir"` + CmdLineOptions []string `yaml:"cmdline_options"` +} + +var NucleiTemplateFail = errors.New("Nuclei template failed") + +func (ts *NucleiConfig) RunNucleiTemplate(test_name string, template_path string, target string) error { + tstamp := time.Now().Unix() + //template_path is the full path to the template, we just want the name ie. "sqli-random-test" + tmp := strings.Split(template_path, "/") + template := strings.Split(tmp[len(tmp)-1], ".")[0] + + output_prefix := fmt.Sprintf("%s/%s_%s-%d", ts.OutputDir, test_name, template, tstamp) + + args := []string{ + "-u", target, + "-t", template_path, + "-o", output_prefix + ".json", + } + args = append(args, ts.CmdLineOptions...) + cmd := exec.Command(ts.Path, args...) + + var out bytes.Buffer + var out_err bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out_err + err := cmd.Run() + if err := os.WriteFile(output_prefix+"_stdout.txt", out.Bytes(), 0644); err != nil { + log.Warningf("Error writing stdout: %s", err) + } + if err := os.WriteFile(output_prefix+"_stderr.txt", out_err.Bytes(), 0644); err != nil { + log.Warningf("Error writing stderr: %s", err) + } + if err != nil { + log.Warningf("Error running nuclei: %s", err) + log.Warningf("Stdout saved to %s", output_prefix+"_stdout.txt") + log.Warningf("Stderr saved to %s", output_prefix+"_stderr.txt") + log.Warningf("Nuclei generated output saved to %s", output_prefix+".json") + return err + } else if len(out.String()) == 0 { + //No stdout means no finding, it means our test failed + return NucleiTemplateFail + } + return nil +} From 94a378d230a1d71f06a1f48e47d9e84b475b04b9 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 17 Nov 2023 18:07:03 +0100 Subject: [PATCH 186/263] up --- pkg/cwhub/items.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index facf53dfb5d..02d438ad787 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -201,6 +201,18 @@ func (i *Item) logMissingSubItems() { } } + for _, subName := range i.WaapConfigs { + if i.hub.GetItem(WAAP_CONFIGS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, WAAP_CONFIGS, i.Name) + } + } + + for _, subName := range i.WaapRules { + if i.hub.GetItem(WAAP_RULES, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, WAAP_RULES, i.Name) + } + } + for _, subName := range i.Collections { if i.hub.GetItem(COLLECTIONS, subName) == nil { log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) From 8173e1ba42be2f42ebbe957819a397e3132cd19b Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Mon, 20 Nov 2023 10:48:10 +0100 Subject: [PATCH 187/263] add timeout to auth request --- pkg/acquisition/modules/waap/waap.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index cf9fd2ad661..5e0174cea59 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -297,16 +297,20 @@ func (w *WaapSource) Dump() interface{} { } func (w *WaapSource) IsAuth(apiKey string) bool { - client := &http.Client{} + client := &http.Client{ + Timeout: 200 * time.Millisecond, + } + req, err := http.NewRequest("HEAD", w.lapiURL, nil) if err != nil { - fmt.Println("Error creating request:", err) + log.Errorf("Error creating request: %s", err) return false } + req.Header.Add("X-Api-Key", apiKey) resp, err := client.Do(req) if err != nil { - fmt.Println("Error performing request:", err) + log.Errorf("Error performing request: %s", err) return false } defer resp.Body.Close() From 6b317f072323823f20abf61b2329297203c15c65 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:41:31 +0100 Subject: [PATCH 188/263] Refact cwhub: simplify tree scan and dependency checks (#2600) * method rename: GetInstalledItemsAsString() -> GetInstalledItemNames() * use path package * Comments and method names * Extract method Item.setVersionState() from Hub.itemVisit() * refact localSync(), itemVisit() etc. * fix check for cyclic dependencies, with test --- cmd/crowdsec-cli/capi.go | 2 +- cmd/crowdsec-cli/console.go | 2 +- cmd/crowdsec-cli/item_suggest.go | 2 +- cmd/crowdsec-cli/lapi.go | 2 +- cmd/crowdsec-cli/support.go | 2 +- cmd/crowdsec/output.go | 4 +- pkg/cwhub/dataset.go | 6 +- pkg/cwhub/enable.go | 14 +- pkg/cwhub/helpers.go | 62 ++++-- pkg/cwhub/hub.go | 31 ++- pkg/cwhub/items.go | 9 +- pkg/cwhub/items_test.go | 5 +- pkg/cwhub/remote.go | 1 + pkg/cwhub/sync.go | 291 +++++++++++++------------- test/bats/20_hub_collections_dep.bats | 10 + 15 files changed, 245 insertions(+), 198 deletions(-) diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 69fc0630dc9..0261eab9cb6 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -156,7 +156,7 @@ func NewCapiStatusCmd() *cobra.Command { return err } - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 79ba6a053d9..1caf11752d3 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -76,7 +76,7 @@ After running this command your will need to validate the enrollment in the weba return err } - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get installed scenarios: %s", err) } diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index c9dc09eba7d..e9db3b7b9df 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -61,7 +61,7 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st return nil, cobra.ShellCompDirectiveDefault } - items, err := hub.GetInstalledItemsAsString(itemType) + items, err := hub.GetInstalledItemNames(itemType) if err != nil { cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) return nil, cobra.ShellCompDirectiveDefault diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 6cfe40a8607..b2870cb200f 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -43,7 +43,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { log.Fatal(err) } - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { log.Fatalf("failed to get scenarios : %s", err) } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 7841e1fc469..1470d37aa96 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -174,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str if err != nil { return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) } - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) } diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 0abb8a9c9d2..b04e849818c 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -71,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky var cache []types.RuntimeAlert var cacheMutex sync.Mutex - scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("loading list of installed hub scenarios: %w", err) } @@ -94,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky URL: apiURL, PapiURL: papiURL, VersionPrefix: "v1", - UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, + UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)}, }) if err != nil { return fmt.Errorf("new client api: %w", err) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index a0e710ebd1e..f002c668edd 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -18,6 +18,7 @@ type DataSet struct { Data []types.DataSource `yaml:"data,omitempty"` } +// downloadFile downloads a file and writes it to disk, with no hash verification func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) @@ -37,6 +38,7 @@ func downloadFile(url string, destPath string) error { } defer file.Close() + // avoid reading the whole file in memory _, err = io.Copy(file, resp.Body) if err != nil { return err @@ -49,8 +51,8 @@ func downloadFile(url string, destPath string) error { return nil } -// downloadData downloads the data files for an item -func downloadData(dataFolder string, force bool, reader io.Reader) error { +// downloadDataSet downloads all the data files for an item +func downloadDataSet(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 49946c8693c..1a5da53bf95 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -10,14 +10,15 @@ import ( log "github.com/sirupsen/logrus" ) -// installLink returns the location of the symlink to the actual config file (eg. /etc/crowdsec/collections/xyz.yaml) -func (i *Item) installLink() string { +// installLink returns the location of the symlink to the downloaded config file +// (eg. /etc/crowdsec/collections/xyz.yaml) +func (i *Item) installLinkPath() string { return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName) } // makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir func (i *Item) createInstallLink() error { - dest, err := filepath.Abs(i.installLink()) + dest, err := filepath.Abs(i.installLinkPath()) if err != nil { return err } @@ -102,8 +103,9 @@ func (i *Item) purge() error { return nil } +// removeInstallLink removes the symlink to the downloaded content func (i *Item) removeInstallLink() error { - syml, err := filepath.Abs(i.installLink()) + syml, err := filepath.Abs(i.installLinkPath()) if err != nil { return err } @@ -143,13 +145,13 @@ func (i *Item) removeInstallLink() error { return nil } -// disable removes the symlink to the downloaded content, also removes the content if purge is true +// disable removes the install link, and optionally the downloaded content func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not err := i.removeInstallLink() if os.IsNotExist(err) { if !purge && !force { - return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLink()) + return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLinkPath()) } } else if err != nil { return err diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 6d106f813c9..320d5a5e84a 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -52,20 +52,46 @@ func (i *Item) Install(force bool, downloadOnly bool) error { return nil } -// allDependencies return a list of all dependencies and sub-dependencies of the item -func (i *Item) allDependencies() []*Item { - var deps []*Item +// allDependencies returns a list of all (direct or indirect) dependencies of the item +func (i *Item) allDependencies() ([]*Item, error) { + var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error - for _, dep := range i.SubItems() { - if dep == i { - log.Errorf("circular dependency detected: %s depends on %s", dep.Name, i.Name) - continue + collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error { + if item == nil { + return nil + } + + if visited[item] { + return nil + } + + visited[item] = true + + for _, subItem := range item.SubItems() { + if subItem == i { + return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name) + } + + *result = append(*result, subItem) + + err := collectSubItems(subItem, visited, result) + if err != nil { + return err + } } - deps = append(deps, dep.allDependencies()...) + return nil + } + + ret := []*Item{} + visited := map[*Item]bool{} + + err := collectSubItems(i, visited, &ret) + if err != nil { + return nil, err } - return append(deps, i) + return ret, nil } // Remove disables the item, optionally removing the downloaded content @@ -85,15 +111,18 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { removed := false - allDeps := i.allDependencies() + allDeps, err := i.allDependencies() + if err != nil { + return false, err + } for _, sub := range i.SubItems() { if !sub.Installed { continue } - // if the other collection(s) are direct or indirect dependencies of the current one, it's good to go - // log parent collections + // if the sub depends on a collection that is not a direct or indirect dependency + // of the current item, it is not removed for _, subParent := range sub.parentCollections() { if subParent == i { continue @@ -113,8 +142,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { removed = removed || subRemoved } - err := i.disable(purge, force) - if err != nil { + if err = i.disable(purge, force); err != nil { return false, fmt.Errorf("while removing %s: %w", i.Name, err) } @@ -171,7 +199,7 @@ func (i *Item) Upgrade(force bool) (bool, error) { return updated, nil } -// downloadLatest will download the latest version of Item to the tdir directory +// downloadLatest downloads the latest version of the item to the hub directory func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { // XXX: should return the path of the downloaded file (taken from download()) log.Debugf("Downloading %s %s", i.Type, i.Name) @@ -314,7 +342,7 @@ func (i *Item) download(overwrite bool) error { i.Tainted = false i.UpToDate = true - if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return fmt.Errorf("while downloading data for %s: %w", i.FileName, err) } @@ -332,7 +360,7 @@ func (i *Item) DownloadDataIfNeeded(force bool) error { defer itemFile.Close() - if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil { + if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index d27264694bf..ee3198fbb73 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "path" "strings" log "github.com/sirupsen/logrus" @@ -12,12 +13,10 @@ import ( ) type Hub struct { - Items HubItems - local *csconfig.LocalHubCfg - remote *RemoteHubCfg - skippedLocal int - skippedTainted int - Warnings []string + Items HubItems + local *csconfig.LocalHubCfg + remote *RemoteHubCfg + Warnings []string } func (h *Hub) GetDataDir() string { @@ -82,8 +81,7 @@ func (h *Hub) parseIndex() error { } item.Type = itemType - x := strings.Split(item.RemotePath, "/") - item.FileName = x[len(x)-1] + item.FileName = path.Base(item.RemotePath) item.logMissingSubItems() } @@ -95,6 +93,8 @@ func (h *Hub) parseIndex() error { // ItemStats returns total counts of the hub items func (h *Hub) ItemStats() []string { loaded := "" + local := 0 + tainted := 0 for _, itemType := range ItemTypes { if len(h.Items[itemType]) == 0 { @@ -102,11 +102,20 @@ func (h *Hub) ItemStats() []string { } loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + + for _, item := range h.Items[itemType] { + if item.IsLocal() { + local++ + } + + if item.Tainted { + tainted++ + } + } } loaded = strings.Trim(loaded, ", ") if loaded == "" { - // empty hub loaded = "0 items" } @@ -114,8 +123,8 @@ func (h *Hub) ItemStats() []string { fmt.Sprintf("Loaded: %s", loaded), } - if h.skippedLocal > 0 || h.skippedTainted > 0 { - ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)) + if local > 0 || tainted > 0 { + ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted)) } return ret diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 2ed593e2093..8492a07f349 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -186,6 +186,7 @@ func (i *Item) logMissingSubItems() { } } +// parentCollections returns the list of items (collections) that have this item as a direct dependency func (i *Item) parentCollections() []*Item { ret := make([]*Item, 0) @@ -281,8 +282,8 @@ func (h *Hub) GetItem(itemType string, itemName string) *Item { return h.GetItemMap(itemType)[itemName] } -// GetItemNames returns the list of item (full) names for a given type -// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// GetItemNames returns the list of (full) item names for a given type +// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx // The names can be used to retrieve the item with GetItem() func (h *Hub) GetItemNames(itemType string) []string { m := h.GetItemMap(itemType) @@ -335,8 +336,8 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { return retItems, nil } -// GetInstalledItemsAsString returns the names of the installed items -func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { +// GetInstalledItemNames returns the names of the installed items +func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) { items, err := h.GetInstalledItems(itemType) if err != nil { return nil, err diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index ed11d80ae8d..cb942433031 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -36,7 +36,10 @@ func TestItemStatus(t *testing.T) { } stats := hub.ItemStats() - require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats) + require.Equal(t, []string{ + "Loaded: 2 parsers, 1 scenarios, 3 collections", + "Unmanaged items: 3 local, 0 tainted", + }, stats) } func TestGetters(t *testing.T) { diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index c3855d5e02d..2b395681062 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -17,6 +17,7 @@ type RemoteHubCfg struct { IndexPath string } +// urlTo builds the URL to download a file from the remote hub func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { if r == nil { return "", ErrNilRemoteHub diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 2d55e9e21d6..4bcf6df4419 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -19,21 +19,18 @@ func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } -func handleSymlink(path string) (string, error) { +// linkTarget returns the target of a symlink, or empty string if it's dangling +func linkTarget(path string) (string, error) { hubpath, err := os.Readlink(path) if err != nil { - return "", fmt.Errorf("unable to read symlink of %s", path) + return "", fmt.Errorf("unable to read symlink: %s", path) } - // the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml + + log.Tracef("symlink %s -> %s", path, hubpath) + _, err = os.Lstat(hubpath) if os.IsNotExist(err) { - log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath) - // remove the symlink - if err = os.Remove(path); err != nil { - return "", fmt.Errorf("failed to unlink %s: %w", path, err) - } - - // ignore this file + log.Infof("link target does not exist: %s -> %s", path, hubpath) return "", nil } @@ -57,15 +54,15 @@ func getSHA256(filepath string) (string, error) { } type itemFileInfo struct { + inhub bool fname string stage string ftype string fauthor string } -func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { - ret := itemFileInfo{} - inhub := false +func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { + var ret *itemFileInfo hubDir := h.local.HubDir installDir := h.local.InstallDir @@ -78,37 +75,41 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { if strings.HasPrefix(path, hubDir) { log.Tracef("in hub dir") - inhub = true //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { - return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } - ret.fname = subs[len(subs)-1] - ret.fauthor = subs[len(subs)-2] - ret.stage = subs[len(subs)-3] - ret.ftype = subs[len(subs)-4] + ret = &itemFileInfo{ + inhub: true, + fname: subs[len(subs)-1], + fauthor: subs[len(subs)-2], + stage: subs[len(subs)-3], + ftype: subs[len(subs)-4], + } } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { - return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) + return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ///.../config/parser/stage/file.yaml ///.../config/postoverflow/stage/file.yaml ///.../config/scenarios/scenar.yaml ///.../config/collections/linux.yaml //file is empty - ret.fname = subs[len(subs)-1] - ret.stage = subs[len(subs)-2] - ret.ftype = subs[len(subs)-3] - ret.fauthor = "" + ret = &itemFileInfo{ + inhub: false, + fname: subs[len(subs)-1], + stage: subs[len(subs)-2], + ftype: subs[len(subs)-3], + fauthor: "", + } } else { - return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) + return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) } log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) - // log.Infof("%s -> name:%s stage:%s", path, fname, stage) if ret.stage == SCENARIOS { ret.ftype = SCENARIOS @@ -118,15 +119,15 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { ret.stage = "" } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { // it's a PARSER / POSTOVERFLOW with a stage - return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) + return nil, fmt.Errorf("unknown configuration type for file '%s'", path) } log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) - return ret, inhub, nil + return ret, nil } -// sortedVersions returns the input data, sorted in reverse order by semver +// sortedVersions returns the input data, sorted in reverse order (new, old) by semver func sortedVersions(raw []string) ([]string, error) { vs := make([]*semver.Version, len(raw)) @@ -149,8 +150,22 @@ func sortedVersions(raw []string) ([]string, error) { return ret, nil } +func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item { + _, fileName := filepath.Split(path) + + return &Item{ + hub: h, + Name: info.fname, + Stage: info.stage, + Installed: true, + Type: info.ftype, + LocalPath: path, + UpToDate: true, + FileName: fileName, + } +} + func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { - local := false hubpath := "" if err != nil { @@ -159,89 +174,59 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { return nil } + // only happens if the current working directory was removed (!) path, err = filepath.Abs(path) if err != nil { return err } - // we only care about files - if f == nil || f.IsDir() { + // we only care about YAML files + if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) { return nil } - if !isYAMLFileName(f.Name()) { - return nil - } - - info, inhub, err := h.getItemInfo(path) + info, err := h.getItemFileInfo(path) if err != nil { return err } - /* - we can encounter 'collections' in the form of a symlink: - /etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml - when the collection is installed, both files are created - */ // non symlinks are local user files or hub files if f.Type()&os.ModeSymlink == 0 { - local = true + log.Tracef("%s is not a symlink", path) + + if !info.inhub { + log.Tracef("%s is a local file, skip", path) + h.Items[info.ftype][info.fname] = newLocalItem(h, path, info) - log.Tracef("%s isn't a symlink", path) + return nil + } } else { - hubpath, err = handleSymlink(path) + hubpath, err = linkTarget(path) if err != nil { return err } - log.Tracef("%s points to %s", path, hubpath) if hubpath == "" { - // ignore this file + // target does not exist, the user might have removed the file + // or switched to a hub branch without it return nil } } - // if it's not a symlink and not in hub, it's a local file, don't bother - if local && !inhub { - log.Tracef("%s is a local file, skip", path) - h.skippedLocal++ - - _, fileName := filepath.Split(path) - - h.Items[info.ftype][info.fname] = &Item{ - hub: h, - Name: info.fname, - Stage: info.stage, - Installed: true, - Type: info.ftype, - LocalPath: path, - UpToDate: true, - FileName: fileName, - } - - return nil - } - // try to find which configuration item it is log.Tracef("check [%s] of %s", info.fname, info.ftype) - match := false - for name, item := range h.Items[info.ftype] { - log.Tracef("check [%s] vs [%s]: %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") - if info.fname != item.FileName { - log.Tracef("%s != %s (filename)", info.fname, item.FileName) continue } - // wrong stage if item.Stage != info.stage { continue } // if we are walking hub dir, just mark present files as downloaded - if inhub { + if info.inhub { // wrong author if info.fauthor != item.Author { continue @@ -262,66 +247,9 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { continue } - sha, err := getSHA256(path) + err := item.setVersionState(path, info.inhub) if err != nil { - log.Fatalf("Failed to get sha of %s: %v", path, err) - } - - // let's reverse sort the versions to deal with hash collisions (#154) - versions := make([]string, 0, len(item.Versions)) - for k := range item.Versions { - versions = append(versions, k) - } - - versions, err = sortedVersions(versions) - if err != nil { - return fmt.Errorf("while syncing %s %s: %w", info.ftype, info.fname, err) - } - - for _, version := range versions { - if item.Versions[version].Digest != sha { - continue - } - - // we got an exact match, update struct - - item.Downloaded = true - item.LocalHash = sha - - if !inhub { - log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version) - item.LocalPath = path - item.LocalVersion = version - item.Tainted = false - // if we're walking the hub, present file doesn't means installed file - item.Installed = true - } - - if version == item.Version { - log.Tracef("%s is up-to-date", item.Name) - item.UpToDate = true - } - - match = true - - break - } - - if !match { - log.Tracef("got tainted match for %s: %s", item.Name, path) - - h.skippedTainted++ - - // the file and the stage is right, but the hash is wrong, it has been tainted by user - if !inhub { - item.LocalPath = path - item.Installed = true - } - - item.UpToDate = false - item.LocalVersion = "?" - item.Tainted = true - item.LocalHash = sha + return err } h.Items[info.ftype][name] = item @@ -365,11 +293,13 @@ func (h *Hub) checkSubItems(v *Item) error { if sub.Tainted { v.Tainted = true + // XXX: improve msg return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) } if !sub.Installed && v.Installed { v.Tainted = true + // XXX: improve msg return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) } @@ -388,9 +318,8 @@ func (h *Hub) checkSubItems(v *Item) error { return nil } -func (h *Hub) syncDir(dir string) ([]string, error) { - warnings := []string{} - +// syncDir scans a directory for items, and updates the Hub state accordingly +func (h *Hub) syncDir(dir string) error { // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { // cpath: top-level item directory, either downloaded or installed items. @@ -408,11 +337,31 @@ func (h *Hub) syncDir(dir string) ([]string, error) { } if err = filepath.WalkDir(cpath, h.itemVisit); err != nil { - return warnings, err + return err } } + return nil +} + +// localSync updates the hub state with downloaded, installed and local items +func (h *Hub) localSync() error { + err := h.syncDir(h.local.InstallDir) + if err != nil { + return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + } + + if err = h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + } + + warnings := make([]string, 0) + for _, item := range h.Items[COLLECTIONS] { + if _, err := item.allDependencies(); err != nil { + return err + } + if !item.Installed { continue } @@ -434,27 +383,69 @@ func (h *Hub) syncDir(dir string) ([]string, error) { log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) } - return warnings, nil + h.Warnings = warnings + + return nil } -// Updates the info from HubInit() with the local state -func (h *Hub) localSync() error { - h.skippedLocal = 0 - h.skippedTainted = 0 - h.Warnings = []string{} +func (i *Item) setVersionState(path string, inhub bool) error { + var err error - warnings, err := h.syncDir(h.local.InstallDir) + i.LocalHash, err = getSHA256(path) if err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + return fmt.Errorf("failed to get sha256 of %s: %w", path, err) } - h.Warnings = append(h.Warnings, warnings...) + // let's reverse sort the versions to deal with hash collisions (#154) + versions := make([]string, 0, len(i.Versions)) + for k := range i.Versions { + versions = append(versions, k) + } - if warnings, err = h.syncDir(h.local.HubDir); err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + versions, err = sortedVersions(versions) + if err != nil { + return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err) + } + + i.LocalVersion = "?" + + for _, version := range versions { + if i.Versions[version].Digest == i.LocalHash { + i.LocalVersion = version + break + } + } + + if i.LocalVersion == "?" { + log.Tracef("got tainted match for %s: %s", i.Name, path) + + if !inhub { + i.LocalPath = path + i.Installed = true + } + + i.UpToDate = false + i.Tainted = true + + return nil } - h.Warnings = append(h.Warnings, warnings...) + // we got an exact match, update struct + + i.Downloaded = true + + if !inhub { + log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.LocalVersion, i.Version) + i.LocalPath = path + i.Tainted = false + // if we're walking the hub, present file doesn't means installed file + i.Installed = true + } + + if i.LocalVersion == i.Version { + log.Tracef("%s is up-to-date", i.Name) + i.UpToDate = true + } return nil } diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats index a44d8bc9c78..d7983aeab50 100644 --- a/test/bats/20_hub_collections_dep.bats +++ b/test/bats/20_hub_collections_dep.bats @@ -112,3 +112,13 @@ teardown() { rune -0 cscli parsers list -o json rune -0 jq -e '.parsers | length == 0' <(output) } + +@test "cscli collections (dependencies IV: looper)" { + hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{collections:["crowdsecurity/linux"]}}}') + echo "$hub_dep" >"$INDEX_PATH" + + rune -1 cscli hub list + assert_stderr --partial "circular dependency detected" + rune -1 wait-for "${CROWDSEC}" + assert_stderr --partial "circular dependency detected" +} From 4a265ca4afa765a000d4dadfec84fdca0fb728e1 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 20 Nov 2023 13:27:46 +0100 Subject: [PATCH 189/263] up --- pkg/cwhub/sync.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 9651fe2b26d..d9e08cd649e 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -10,11 +10,17 @@ import ( "sort" "strings" + "slices" + "github.com/Masterminds/semver/v3" log "github.com/sirupsen/logrus" - "slices" + "gopkg.in/yaml.v3" ) +type localItem struct { + Name string `yaml:"name"` +} + func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } @@ -214,9 +220,22 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { _, fileName := filepath.Split(path) + item := localItem{} + itemContent, err := os.ReadFile(path) + + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + + err = yaml.Unmarshal(itemContent, &item) + + if err != nil { + return fmt.Errorf("failed to unmarshal %s: %w", path, err) + } + h.Items[info.ftype][info.fname] = &Item{ hub: h, - Name: info.fname, + Name: item.Name, Stage: info.stage, Installed: true, Type: info.ftype, From 2d01e4680f740d74f9629f602a4accdd79f03468 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 20 Nov 2023 14:25:33 +0100 Subject: [PATCH 190/263] do not error if no waap rules are present --- pkg/hubtest/hubtest_item.go | 8 ++++++-- pkg/waf/loader.go | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 65c95a2bc15..b1a6c30ad69 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -827,8 +827,8 @@ func (t *HubTestItem) Run() error { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) } - // copy the waap-config file and acquis *only* if nuclei template is set -> it means we're testing waap - if t.Config.NucleiTemplate != "" { + //if it's a waap rule test, we need acquis and waap profile + if len(t.Config.WaapRules) > 0 { // copy template acquis file to runtime folder log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath) if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil { @@ -840,6 +840,10 @@ func (t *HubTestItem) Run() error { if err = Copy(t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml")); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml"), err) } + } else { //otherwise we drop a blank acquis file + if err = os.WriteFile(t.RuntimeAcquisFilePath, []byte(""), os.ModePerm); err != nil { + return fmt.Errorf("unable to write blank acquis file '%s': %s", t.RuntimeAcquisFilePath, err) + } } // install the hub in the runtime folder diff --git a/pkg/waf/loader.go b/pkg/waf/loader.go index ffddea121e5..43c56411c76 100644 --- a/pkg/waf/loader.go +++ b/pkg/waf/loader.go @@ -1,7 +1,6 @@ package waf import ( - "fmt" "os" "github.com/crowdsecurity/crowdsec/pkg/cwhub" @@ -53,7 +52,7 @@ func LoadWaapRules(hubInstance *cwhub.Hub) error { } if len(waapRules) == 0 { - return fmt.Errorf("no waap rules found in hub") + log.Debugf("No waap rules found") } return nil } From 7b1074f0cb17498be28bea2d85cc9916e3c6ffcb Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:58:42 +0100 Subject: [PATCH 191/263] Refact cwhub (#2603) * Split RemoteHub.downloadIndex() = Hub.updateIndex() + RemoteHub.fetchIndex() * Functions safePath(), Item.installPath(), item.downloadPath() --- pkg/cwhub/cwhub.go | 22 ++++++++++++++ pkg/cwhub/dataset.go | 6 ++-- pkg/cwhub/enable.go | 40 ++++++++++++++++++------ pkg/cwhub/enable_test.go | 4 +-- pkg/cwhub/helpers.go | 66 +++++++++++++++++++--------------------- pkg/cwhub/hub.go | 45 +++++++++++++++++++++------ pkg/cwhub/hub_test.go | 12 +++++--- pkg/cwhub/remote.go | 38 ++++++----------------- pkg/cwhub/sync.go | 7 ++++- 9 files changed, 150 insertions(+), 90 deletions(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index c7e17bd62f3..a2d10e5aa6f 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -5,10 +5,32 @@ package cwhub import ( + "fmt" "net/http" + "path/filepath" + "strings" "time" ) var hubClient = &http.Client{ Timeout: 120 * time.Second, } + +// safePath returns an error if the given file path would escape the base directory. +func safePath(dir, filePath string) (string, error) { + absBaseDir, err := filepath.Abs(filepath.Clean(dir)) + if err != nil { + return "", err + } + + absFilePath, err := filepath.Abs(filepath.Join(dir, filePath)) + if err != nil { + return "", err + } + + if !strings.HasPrefix(absFilePath, absBaseDir) { + return "", fmt.Errorf("path %s escapes base directory %s", filePath, dir) + } + + return absFilePath, nil +} diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index f002c668edd..79eb91573b0 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "path/filepath" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -67,7 +66,10 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader) error { } for _, dataS := range data.Data { - destPath := filepath.Join(dataFolder, dataS.DestPath) + destPath, err := safePath(dataFolder, dataS.DestPath) + if err != nil { + return err + } if _, err := os.Stat(destPath); os.IsNotExist(err) || force { log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 1a5da53bf95..b38f74e9740 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -10,15 +10,33 @@ import ( log "github.com/sirupsen/logrus" ) -// installLink returns the location of the symlink to the downloaded config file +// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local // (eg. /etc/crowdsec/collections/xyz.yaml) -func (i *Item) installLinkPath() string { - return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName) +// raises an error if the path goes outside of the install dir +func (i *Item) installPath() (string, error) { + p := i.Type + if i.Stage != "" { + p = filepath.Join(p, i.Stage) + } + + return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName)) +} + +// downloadPath returns the location of the actual config file in the hub +// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml) +// raises an error if the path goes outside of the hub dir +func (i *Item) downloadPath() (string, error) { + ret, err := safePath(i.hub.local.HubDir, i.RemotePath) + if err != nil { + return "", err + } + + return ret, nil } // makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir func (i *Item) createInstallLink() error { - dest, err := filepath.Abs(i.installLinkPath()) + dest, err := i.installPath() if err != nil { return err } @@ -33,7 +51,7 @@ func (i *Item) createInstallLink() error { return nil } - src, err := filepath.Abs(filepath.Join(i.hub.local.HubDir, i.RemotePath)) + src, err := i.downloadPath() if err != nil { return err } @@ -86,7 +104,10 @@ func (i *Item) purge() error { return nil } - src := filepath.Join(i.hub.local.HubDir, i.RemotePath) + src, err := i.downloadPath() + if err != nil { + return err + } if err := os.Remove(src); err != nil { if os.IsNotExist(err) { @@ -105,7 +126,7 @@ func (i *Item) purge() error { // removeInstallLink removes the symlink to the downloaded content func (i *Item) removeInstallLink() error { - syml, err := filepath.Abs(i.installLinkPath()) + syml, err := i.installPath() if err != nil { return err } @@ -126,7 +147,7 @@ func (i *Item) removeInstallLink() error { return fmt.Errorf("while reading symlink: %w", err) } - src, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath) + src, err := i.downloadPath() if err != nil { return err } @@ -151,7 +172,8 @@ func (i *Item) disable(purge bool, force bool) error { err := i.removeInstallLink() if os.IsNotExist(err) { if !purge && !force { - return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLinkPath()) + link, _ := i.installPath() + return fmt.Errorf("link %s does not exist (override with --force or --purge)", link) } } else if err != nil { return err diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 9173024a4b0..fc9863d9ea5 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -10,7 +10,7 @@ import ( func testInstall(hub *Hub, t *testing.T, item *Item) { // Install the parser - err := item.downloadLatest(false, false) + _, err := item.downloadLatest(false, false) require.NoError(t, err, "failed to download %s", item.Name) err = hub.localSync() @@ -48,7 +48,7 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := item.downloadLatest(true, true) + _, err := item.downloadLatest(true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 320d5a5e84a..f014a2a7d1f 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -13,7 +13,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "github.com/enescakir/emoji" log "github.com/sirupsen/logrus" @@ -31,13 +30,14 @@ func (i *Item) Install(force bool, downloadOnly bool) error { } // XXX: confusing semantic between force and updateOnly? - if err := i.downloadLatest(force, true); err != nil { + filePath, err := i.downloadLatest(force, true) + if err != nil { return fmt.Errorf("while downloading %s: %w", i.Name, err) } if downloadOnly { // XXX: should get the path from downloadLatest - log.Infof("Downloaded %s to %s", i.Name, filepath.Join(i.hub.local.HubDir, i.RemotePath)) + log.Infof("Downloaded %s to %s", i.Name, filePath) return nil } @@ -177,7 +177,7 @@ func (i *Item) Upgrade(force bool) (bool, error) { } } - if err := i.downloadLatest(force, true); err != nil { + if _, err := i.downloadLatest(force, true); err != nil { return false, fmt.Errorf("%s: download failed: %w", i.Name, err) } @@ -200,7 +200,7 @@ func (i *Item) Upgrade(force bool) (bool, error) { } // downloadLatest downloads the latest version of the item to the hub directory -func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { +func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { // XXX: should return the path of the downloaded file (taken from download()) log.Debugf("Downloading %s %s", i.Type, i.Name) @@ -216,39 +216,40 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error { if sub.HasSubItems() { log.Tracef("collection, recurse") - if err := sub.downloadLatest(overwrite, updateOnly); err != nil { - return fmt.Errorf("while downloading %s: %w", sub.Name, err) + if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil { + return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) } } downloaded := sub.Downloaded - if err := sub.download(overwrite); err != nil { - return fmt.Errorf("while downloading %s: %w", sub.Name, err) + if _, err := sub.download(overwrite); err != nil { + return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) } // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if sub.Downloaded is false because maybe the item has been disabled by the user. if !sub.Installed && !downloaded { if err := sub.enable(); err != nil { - return fmt.Errorf("enabling '%s': %w", sub.Name, err) + return "", fmt.Errorf("enabling '%s': %w", sub.Name, err) } } } if !i.Installed && updateOnly && i.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) - return nil + return "", nil } - if err := i.download(overwrite); err != nil { - return fmt.Errorf("failed to download item: %w", err) + ret, err := i.download(overwrite) + if err != nil { + return "", fmt.Errorf("failed to download item: %w", err) } - return nil + return ret, nil } -// fetch downloads the item from the hub, verifies the hash and returns the body +// fetch downloads the item from the hub, verifies the hash and returns the content func (i *Item) fetch() ([]byte, error) { url, err := i.hub.remote.urlTo(i.RemotePath) if err != nil { @@ -287,12 +288,12 @@ func (i *Item) fetch() ([]byte, error) { } // download downloads the item from the hub and writes it to the hub directory -func (i *Item) download(overwrite bool) error { +func (i *Item) download(overwrite bool) (string, error) { // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { if i.Tainted { log.Debugf("%s: tainted, not updated", i.Name) - return nil + return "", nil } if i.UpToDate { @@ -303,39 +304,33 @@ func (i *Item) download(overwrite bool) error { body, err := i.fetch() if err != nil { - return err + return "", err } - tdir := i.hub.local.HubDir - - //all good, install - - finalPath, err := filepath.Abs(filepath.Join(tdir, i.RemotePath)) - if err != nil { - return err - } + // all good, install // ensure that target file is within target dir - if !strings.HasPrefix(finalPath, tdir) { - return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir) + finalPath, err := i.downloadPath() + if err != nil { + return "", err } parentDir := filepath.Dir(finalPath) if err = os.MkdirAll(parentDir, os.ModePerm); err != nil { - return fmt.Errorf("while creating %s: %w", parentDir, err) + return "", fmt.Errorf("while creating %s: %w", parentDir, err) } // check actual file if _, err = os.Stat(finalPath); !os.IsNotExist(err) { log.Warningf("%s: overwrite", i.Name) - log.Debugf("target: %s/%s", tdir, i.RemotePath) + log.Debugf("target: %s", finalPath) } else { log.Infof("%s: OK", i.Name) } if err = os.WriteFile(finalPath, body, 0o644); err != nil { - return fmt.Errorf("while writing %s: %w", finalPath, err) + return "", fmt.Errorf("while writing %s: %w", finalPath, err) } i.Downloaded = true @@ -343,15 +338,18 @@ func (i *Item) download(overwrite bool) error { i.UpToDate = true if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { - return fmt.Errorf("while downloading data for %s: %w", i.FileName, err) + return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err) } - return nil + return finalPath, nil } // DownloadDataIfNeeded downloads the data files for the item func (i *Item) DownloadDataIfNeeded(force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", i.hub.local.InstallDir, i.Type, i.Stage, i.FileName) + itemFilePath, err := i.installPath() + if err != nil { + return err + } itemFile, err := os.Open(itemFilePath) if err != nil { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index ee3198fbb73..82be29b01d4 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -1,6 +1,7 @@ package cwhub import ( + "bytes" "encoding/json" "fmt" "os" @@ -24,25 +25,25 @@ func (h *Hub) GetDataDir() string { } // NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state -// It also downloads the index if downloadIndex is true -func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex bool) (*Hub, error) { +// It also downloads the index if updateIndex is true +func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) { if local == nil { return nil, fmt.Errorf("no hub configuration found") } - if downloadIndex { - if err := remote.downloadIndex(local.HubIndexFile); err != nil { + hub := &Hub{ + local: local, + remote: remote, + } + + if updateIndex { + if err := hub.updateIndex(); err != nil { return nil, err } } log.Debugf("loading hub idx %s", local.HubIndexFile) - hub := &Hub{ - local: local, - remote: remote, - } - if err := hub.parseIndex(); err != nil { return nil, fmt.Errorf("failed to load index: %w", err) } @@ -129,3 +130,29 @@ func (h *Hub) ItemStats() []string { return ret } + +// updateIndex downloads the latest version of the index and writes it to disk if it changed +func (h *Hub) updateIndex() error { + body, err := h.remote.fetchIndex() + if err != nil { + return err + } + + oldContent, err := os.ReadFile(h.local.HubIndexFile) + if err != nil { + if !os.IsNotExist(err) { + log.Warningf("failed to read hub index: %s", err) + } + } else if bytes.Equal(body, oldContent) { + log.Info("hub index is up to date") + return nil + } + + if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil { + return fmt.Errorf("failed to write hub index: %w", err) + } + + log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body)) + + return nil +} diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 56e2bf376a7..670f8d84356 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -23,7 +23,7 @@ func TestInitHubUpdate(t *testing.T) { require.NoError(t, err) } -func TestDownloadIndex(t *testing.T) { +func TestUpdateIndex(t *testing.T) { // bad url template fmt.Println("Test 'bad URL'") @@ -42,7 +42,9 @@ func TestDownloadIndex(t *testing.T) { IndexPath: "", } - err = hub.remote.downloadIndex(tmpIndex.Name()) + hub.local.HubIndexFile = tmpIndex.Name() + + err = hub.updateIndex() cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") // bad domain @@ -54,7 +56,7 @@ func TestDownloadIndex(t *testing.T) { IndexPath: ".index.json", } - err = hub.remote.downloadIndex(tmpIndex.Name()) + err = hub.updateIndex() require.NoError(t, err) // XXX: this is not failing // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") @@ -68,6 +70,8 @@ func TestDownloadIndex(t *testing.T) { IndexPath: ".index.json", } - err = hub.remote.downloadIndex("/does/not/exist/index.json") + hub.local.HubIndexFile = "/does/not/exist/index.json" + + err = hub.updateIndex() cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:") } diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 2b395681062..c98dfa8f5c2 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -1,13 +1,9 @@ package cwhub import ( - "bytes" "fmt" "io" "net/http" - "os" - - log "github.com/sirupsen/logrus" ) // RemoteHubCfg contains where to find the remote hub, which branch etc. @@ -31,51 +27,35 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil } -// downloadIndex downloads the latest version of the index -func (r *RemoteHubCfg) downloadIndex(localPath string) error { +// fetchIndex downloads the index from the hub and returns the content +func (r *RemoteHubCfg) fetchIndex() ([]byte, error) { if r == nil { - return ErrNilRemoteHub + return nil, ErrNilRemoteHub } url, err := r.urlTo(r.IndexPath) if err != nil { - return fmt.Errorf("failed to build hub index request: %w", err) + return nil, fmt.Errorf("failed to build hub index request: %w", err) } resp, err := hubClient.Get(url) if err != nil { - return fmt.Errorf("failed http request for hub index: %w", err) + return nil, fmt.Errorf("failed http request for hub index: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { - return IndexNotFoundError{url, r.Branch} + return nil, IndexNotFoundError{url, r.Branch} } - return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) + return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("failed to read request answer for hub index: %w", err) - } - - oldContent, err := os.ReadFile(localPath) - if err != nil { - if !os.IsNotExist(err) { - log.Warningf("failed to read hub index: %s", err) - } - } else if bytes.Equal(body, oldContent) { - log.Info("hub index is up to date") - return nil - } - - if err = os.WriteFile(localPath, body, 0o644); err != nil { - return fmt.Errorf("failed to write hub index: %w", err) + return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } - log.Infof("Wrote index to %s, %d bytes", localPath, len(body)) - - return nil + return body, nil } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 4bcf6df4419..a755e10fec2 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -237,7 +237,12 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { continue } - if path == h.local.HubDir+"/"+item.RemotePath { + src, err := item.downloadPath() + if err != nil { + return err + } + + if path == src { log.Tracef("marking %s as downloaded", item.Name) item.Downloaded = true } From 1509c2d97c75be7f8a74186b356b30a3ec1d5f7a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:06:59 +0100 Subject: [PATCH 192/263] pkg/cwhub refact (#2606) * Separate Item and ItemState; fill BelongsToCollections with all ancestors and for uninstalled items too * fix "installed parents" check when removing an item * keep BelongsToCollections in order (case insensitive) --- cmd/crowdsec-cli/config_backup.go | 16 ++--- cmd/crowdsec-cli/itemcommands.go | 20 +++++- cmd/crowdsec-cli/items.go | 8 +-- cmd/crowdsec-cli/simulation.go | 2 +- cmd/crowdsec-cli/utils_table.go | 2 +- cmd/crowdsec/main.go | 4 +- pkg/cwhub/cwhub.go | 2 +- pkg/cwhub/enable.go | 14 ++-- pkg/cwhub/enable_test.go | 32 ++++----- pkg/cwhub/helpers.go | 44 ++++++------ pkg/cwhub/helpers_test.go | 86 ++++++++++++------------ pkg/cwhub/hub.go | 2 +- pkg/cwhub/items.go | 87 +++++++++++++++--------- pkg/cwhub/items_test.go | 16 ++--- pkg/cwhub/sync.go | 108 +++++++++++++++++------------- pkg/hubtest/hubtest_item.go | 6 +- pkg/leakybucket/manager_load.go | 4 +- pkg/parser/unix_parser.go | 4 +- test/bats/20_hub_items.bats | 13 ++++ 19 files changed, 270 insertions(+), 200 deletions(-) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 30a70830ec1..93772d611c7 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -40,13 +40,13 @@ func backupHub(dirPath string) error { clog = clog.WithFields(log.Fields{ "file": v.Name, }) - if !v.Installed { //only backup installed ones + if !v.State.Installed { //only backup installed ones clog.Debugf("[%s] : not installed", k) continue } //for the local/tainted ones, we back up the full file - if v.Tainted || v.IsLocal() || !v.UpToDate { + if v.State.Tainted || v.IsLocal() || !v.State.UpToDate { //we need to backup stages for parsers if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) @@ -54,16 +54,16 @@ func backupHub(dirPath string) error { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) } } - clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.IsLocal(), v.UpToDate) + clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) + if err = CopyFile(v.State.LocalPath, tfile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err) } - clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) + clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile) continue } - clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) + clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate) + clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate) upstreamParsers = append(upstreamParsers, v.Name) } //write the upstream items diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index de7aab68ac1..f36f52a48ca 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -264,6 +264,19 @@ func NewItemsInstallCmd(typeName string) *cobra.Command { return cmd } +// return the names of the installed parents of an item, used to check if we can remove it +func istalledParentNames(item *cwhub.Item) []string { + ret := make([]string, 0) + + for _, parent := range item.ParentCollections() { + if parent.State.Installed { + ret = append(ret, parent.Name) + } + } + + return ret +} + func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { run := func(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -331,10 +344,11 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e return fmt.Errorf("can't find '%s' in %s", itemName, it.name) } - if !force && len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections) - log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + parents := istalledParentNames(item) + if !force && len(parents) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, parents) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) continue } diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index aeefd426303..d445d19364c 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -41,7 +41,7 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b for _, itemName := range itemNames { item := hub.GetItem(itemType, itemName) - if installedOnly && !item.Installed { + if installedOnly && !item.State.Installed { continue } @@ -78,8 +78,8 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item status, emo := item.Status() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, - LocalVersion: item.LocalVersion, - LocalPath: item.LocalPath, + LocalVersion: item.State.LocalVersion, + LocalPath: item.State.LocalPath, Description: item.Description, Status: status, UTF8Status: fmt.Sprintf("%v %s", emo, status), @@ -111,7 +111,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item row := []string{ item.Name, status, - item.LocalVersion, + item.State.LocalVersion, item.Description, } if len(itemTypes) > 1 { diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index c43e22a68da..27aea5831d0 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -154,7 +154,7 @@ func NewSimulationEnableCmd() *cobra.Command { log.Errorf("'%s' doesn't exist or is not a scenario", scenario) continue } - if !item.Installed { + if !item.State.Installed { log.Warningf("'%s' isn't enabled", scenario) } isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario) diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index b4f86718e40..ee3915c7300 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -19,7 +19,7 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) { for _, item := range items { status, emo := item.Status() - t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 61bbfad86ff..8c7fb2991ff 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -81,8 +81,8 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { files []string ) for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) { - if hubScenarioItem.Installed { - files = append(files, hubScenarioItem.LocalPath) + if hubScenarioItem.State.Installed { + files = append(files, hubScenarioItem.State.LocalPath) } } buckets = leakybucket.NewBuckets() diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index a2d10e5aa6f..85569c64e3d 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -16,7 +16,7 @@ var hubClient = &http.Client{ Timeout: 120 * time.Second, } -// safePath returns an error if the given file path would escape the base directory. +// safePath returns a joined path and ensures that it does not escape the base directory. func safePath(dir, filePath string) (string, error) { absBaseDir, err := filepath.Abs(filepath.Clean(dir)) if err != nil { diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index b38f74e9740..2a17fc7243a 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -65,8 +65,8 @@ func (i *Item) createInstallLink() error { // enable enables the item by creating a symlink to the downloaded content, and also enables sub-items func (i *Item) enable() error { - if i.Installed { - if i.Tainted { + if i.State.Installed { + if i.State.Tainted { return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) } @@ -75,7 +75,7 @@ func (i *Item) enable() error { } // if it's a collection, check sub-items even if the collection file itself is up-to-date - if i.UpToDate && !i.HasSubItems() { + if i.State.UpToDate && !i.HasSubItems() { log.Tracef("%s is installed and up-to-date, skip.", i.Name) return nil } @@ -92,14 +92,14 @@ func (i *Item) enable() error { } log.Infof("Enabled %s: %s", i.Type, i.Name) - i.Installed = true + i.State.Installed = true return nil } // purge removes the actual config file that was downloaded func (i *Item) purge() error { - if !i.Downloaded { + if !i.State.Downloaded { log.Infof("removing %s: not downloaded -- no need to remove", i.Name) return nil } @@ -118,7 +118,7 @@ func (i *Item) purge() error { return fmt.Errorf("while removing file: %w", err) } - i.Downloaded = false + i.State.Downloaded = false log.Infof("Removed source file [%s]: %s", i.Name, src) return nil @@ -179,7 +179,7 @@ func (i *Item) disable(purge bool, force bool) error { return err } - i.Installed = false + i.State.Installed = false if purge { if err := i.purge(); err != nil { diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index fc9863d9ea5..35e56915b36 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) err = item.enable() require.NoError(t, err, "failed to enable %s", item.Name) @@ -26,14 +26,14 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) } func testTaint(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) // truncate the file - f, err := os.Create(item.LocalPath) + f, err := os.Create(item.State.LocalPath) require.NoError(t, err) f.Close() @@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name) } func testUpdate(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status _, err := item.downloadLatest(true, true) @@ -55,12 +55,12 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) } func testDisable(hub *Hub, t *testing.T, item *Item) { - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) // Remove err := item.disable(false, false) @@ -71,9 +71,9 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name) // Purge err = item.disable(true, false) @@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index f014a2a7d1f..4a03a12d964 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -21,7 +21,7 @@ import ( // Install installs the item from the hub, downloading it if needed func (i *Item) Install(force bool, downloadOnly bool) error { - if downloadOnly && i.Downloaded && i.UpToDate { + if downloadOnly && i.State.Downloaded && i.State.UpToDate { log.Infof("%s is already downloaded and up-to-date", i.Name) if !force { @@ -100,11 +100,11 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) } - if i.Tainted && !force { + if i.State.Tainted && !force { return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) } - if !i.Installed && !purge { + if !i.State.Installed && !purge { log.Infof("removing %s: not installed -- no need to remove", i.Name) return false, nil } @@ -117,13 +117,17 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { } for _, sub := range i.SubItems() { - if !sub.Installed { + if !sub.State.Installed { continue } // if the sub depends on a collection that is not a direct or indirect dependency // of the current item, it is not removed - for _, subParent := range sub.parentCollections() { + for _, subParent := range sub.ParentCollections() { + if !purge && !subParent.State.Installed { + continue + } + if subParent == i { continue } @@ -156,15 +160,15 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { func (i *Item) Upgrade(force bool) (bool, error) { updated := false - if !i.Downloaded { + if !i.State.Downloaded { return false, fmt.Errorf("can't upgrade %s: not installed", i.Name) } - if !i.Installed { + if !i.State.Installed { return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name) } - if i.UpToDate { + if i.State.UpToDate { log.Infof("%s: up-to-date", i.Name) if err := i.DownloadDataIfNeeded(force); err != nil { @@ -181,8 +185,8 @@ func (i *Item) Upgrade(force bool) (bool, error) { return false, fmt.Errorf("%s: download failed: %w", i.Name, err) } - if !i.UpToDate { - if i.Tainted { + if !i.State.UpToDate { + if i.State.Tainted { log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) } else if i.IsLocal() { log.Infof("%v %s is local", emoji.Prohibited, i.Name) @@ -205,12 +209,12 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { log.Debugf("Downloading %s %s", i.Type, i.Name) for _, sub := range i.SubItems() { - if !sub.Installed && updateOnly && sub.Downloaded { + if !sub.State.Installed && updateOnly && sub.State.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) continue } - log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly) + log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly) // recurse as it's a collection if sub.HasSubItems() { @@ -221,7 +225,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { } } - downloaded := sub.Downloaded + downloaded := sub.State.Downloaded if _, err := sub.download(overwrite); err != nil { return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) @@ -229,14 +233,14 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if sub.Downloaded is false because maybe the item has been disabled by the user. - if !sub.Installed && !downloaded { + if !sub.State.Installed && !downloaded { if err := sub.enable(); err != nil { return "", fmt.Errorf("enabling '%s': %w", sub.Name, err) } } } - if !i.Installed && updateOnly && i.Downloaded { + if !i.State.Installed && updateOnly && i.State.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) return "", nil } @@ -291,12 +295,12 @@ func (i *Item) fetch() ([]byte, error) { func (i *Item) download(overwrite bool) (string, error) { // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { - if i.Tainted { + if i.State.Tainted { log.Debugf("%s: tainted, not updated", i.Name) return "", nil } - if i.UpToDate { + if i.State.UpToDate { // We still have to check if data files are present log.Debugf("%s: up-to-date, not updated", i.Name) } @@ -333,9 +337,9 @@ func (i *Item) download(overwrite bool) (string, error) { return "", fmt.Errorf("while writing %s: %w", finalPath, err) } - i.Downloaded = true - i.Tainted = false - i.UpToDate = true + i.State.Downloaded = true + i.State.Tainted = false + i.State.UpToDate = true if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 0ad647b9917..58d1a56cc82 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -14,16 +14,16 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) // This is the scenario that gets added in next version of collection require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) @@ -44,10 +44,10 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") didUpdate, err := item.Upgrade(false) @@ -55,8 +55,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { require.True(t, didUpdate) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } // Install a collection, disable a scenario. @@ -65,18 +65,18 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -92,11 +92,11 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) @@ -107,7 +107,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test @@ -125,18 +125,18 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -152,12 +152,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario @@ -167,7 +167,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) hub = getHubOrFail(t, hub.local, remote) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") @@ -176,8 +176,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 82be29b01d4..01f7eef0426 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -109,7 +109,7 @@ func (h *Hub) ItemStats() []string { local++ } - if item.Tainted { + if item.State.Tainted { tainted++ } } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 8492a07f349..7e0a10f4c3a 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -39,35 +39,41 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty"` } +// ItemState is used to keep the local state (i.e. at runtime) of an item +// This data is not stored in the index, but is displayed in the output of "cscli ... inspect" +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` // the local meow + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` // has it been locally modified? + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any +} + // Item represents an object managed in the hub. It can be a parser, scenario, collection.. type Item struct { // back pointer to the hub, to retrieve subitems and call install/remove methods hub *Hub + // local (deployed) info + State ItemState + // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json - Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any + Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" + FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json // remote (hub) info RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml Version string `json:"version,omitempty"` // the last version Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} - LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed"` - Downloaded bool `json:"downloaded"` - UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` // has it been locally modified? - // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` @@ -80,7 +86,7 @@ func (i *Item) HasSubItems() bool { } func (i *Item) IsLocal() bool { - return i.Installed && !i.Downloaded + return i.State.Installed && !i.State.Downloaded } // MarshalJSON is used to add the "local" field to the json output @@ -91,10 +97,27 @@ func (i Item) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Alias - Local bool `json:"local"` + // we have to repeat the fields here, json will have inline support in v2 + LocalPath string `json:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + Local bool `json:"local"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty"` }{ - Alias: Alias(i), - Local: i.IsLocal(), + Alias: Alias(i), + LocalPath: i.State.LocalPath, + LocalVersion: i.State.LocalVersion, + LocalHash: i.State.LocalHash, + Installed: i.State.Installed, + Downloaded: i.State.Downloaded, + UpToDate: i.State.UpToDate, + Tainted: i.State.Tainted, + BelongsToCollections: i.State.BelongsToCollections, + Local: i.IsLocal(), }) } @@ -106,9 +129,11 @@ func (i Item) MarshalYAML() (interface{}, error) { return &struct { Alias `yaml:",inline"` - Local bool `yaml:"local"` + State ItemState `yaml:",inline"` + Local bool `yaml:"local"` }{ Alias: Alias(i), + State: i.State, Local: i.IsLocal(), }, nil } @@ -186,11 +211,11 @@ func (i *Item) logMissingSubItems() { } } -// parentCollections returns the list of items (collections) that have this item as a direct dependency -func (i *Item) parentCollections() []*Item { +// ParentCollections returns the list of items (collections) that have this item as a direct dependency +func (i *Item) ParentCollections() []*Item { ret := make([]*Item, 0) - for _, parentName := range i.BelongsToCollections { + for _, parentName := range i.State.BelongsToCollections { parent := i.hub.GetItem(COLLECTIONS, parentName) if parent == nil { continue @@ -208,7 +233,7 @@ func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false - if i.Installed { + if i.State.Installed { ok = true status = "enabled" } @@ -220,10 +245,10 @@ func (i *Item) Status() (string, emoji.Emoji) { } warning := false - if i.Tainted { + if i.State.Tainted { warning = true status += ",tainted" - } else if !i.UpToDate && !i.IsLocal() { + } else if !i.State.UpToDate && !i.IsLocal() { warning = true status += ",update-available" } @@ -233,7 +258,7 @@ func (i *Item) Status() (string, emoji.Emoji) { switch { case !managed: emo = emoji.House - case !i.Installed: + case !i.State.Installed: emo = emoji.Prohibited case warning: emo = emoji.Warning @@ -246,7 +271,7 @@ func (i *Item) Status() (string, emoji.Emoji) { // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { - local, err := semver.NewVersion(i.LocalVersion) + local, err := semver.NewVersion(i.State.LocalVersion) if err != nil { return VersionUnknown } @@ -328,7 +353,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { retItems := make([]*Item, 0) for _, item := range items { - if item.Installed { + if item.State.Installed { retItems = append(retItems, item) } } diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index cb942433031..caffb2eb7a0 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -18,18 +18,18 @@ func TestItemStatus(t *testing.T) { item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) - item.Installed = true - item.UpToDate = false - item.Tainted = false - item.Downloaded = true + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = true txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) - item.Installed = true - item.UpToDate = false - item.Tainted = false - item.Downloaded = false + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = false txt, _ = item.Status() require.Equal(t, "enabled,local", txt) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index a755e10fec2..4325017b4a0 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -12,7 +12,6 @@ import ( "github.com/Masterminds/semver/v3" log "github.com/sirupsen/logrus" - "slices" ) func isYAMLFileName(path string) bool { @@ -53,6 +52,7 @@ func getSHA256(filepath string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } +// information used to create a new Item, from a file path type itemFileInfo struct { inhub bool fname string @@ -154,14 +154,16 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item { _, fileName := filepath.Split(path) return &Item{ - hub: h, - Name: info.fname, - Stage: info.stage, - Installed: true, - Type: info.ftype, - LocalPath: path, - UpToDate: true, - FileName: fileName, + hub: h, + Name: info.fname, + Stage: info.stage, + Type: info.ftype, + FileName: fileName, + State: ItemState{ + LocalPath: path, + Installed: true, + UpToDate: true, + }, } } @@ -244,7 +246,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { if path == src { log.Tracef("marking %s as downloaded", item.Name) - item.Downloaded = true + item.State.Downloaded = true } } else if !hasPathSuffix(hubpath, item.RemotePath) { // wrong file @@ -279,45 +281,41 @@ func (h *Hub) checkSubItems(v *Item) error { } // ensure all the sub-items are installed, or tag the parent as tainted - log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) + log.Tracef("checking submembers of %s installed:%t", v.Name, v.State.Installed) for _, sub := range v.SubItems() { - log.Tracef("check %s installed:%t", sub.Name, sub.Installed) + log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) - if !v.Installed { + if !v.State.Installed { continue } if err := h.checkSubItems(sub); err != nil { - if sub.Tainted { - v.Tainted = true + if sub.State.Tainted { + v.State.Tainted = true } return fmt.Errorf("sub collection %s is broken: %w", sub.Name, err) } - if sub.Tainted { - v.Tainted = true + if sub.State.Tainted { + v.State.Tainted = true // XXX: improve msg return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) } - if !sub.Installed && v.Installed { - v.Tainted = true + if !sub.State.Installed && v.State.Installed { + v.State.Tainted = true // XXX: improve msg return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) } - if !sub.UpToDate { - v.UpToDate = false + if !sub.State.UpToDate { + v.State.UpToDate = false return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) } - if !slices.Contains(sub.BelongsToCollections, v.Name) { - sub.BelongsToCollections = append(sub.BelongsToCollections, v.Name) - } - - log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) + log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.State.Tainted, v.State.UpToDate) } return nil @@ -349,6 +347,15 @@ func (h *Hub) syncDir(dir string) error { return nil } +// insert a string in a sorted slice, case insensitive, and return the new slice +func insertInOrderNoCase(sl []string, value string) []string { + i := sort.Search(len(sl), func(i int) bool { + return strings.ToLower(sl[i]) >= strings.ToLower(value) + }) + + return append(sl[:i], append([]string{value}, sl[i:]...)...) +} + // localSync updates the hub state with downloaded, installed and local items func (h *Hub) localSync() error { err := h.syncDir(h.local.InstallDir) @@ -363,11 +370,18 @@ func (h *Hub) localSync() error { warnings := make([]string, 0) for _, item := range h.Items[COLLECTIONS] { - if _, err := item.allDependencies(); err != nil { + // check for cyclic dependencies + subs, err := item.allDependencies() + if err != nil { return err } - if !item.Installed { + // populate the sub- and sub-sub-items with the collections they belong to + for _, sub := range subs { + sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name) + } + + if !item.State.Installed { continue } @@ -378,14 +392,14 @@ func (h *Hub) localSync() error { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) } case VersionUpdateAvailable: // not up-to-date - warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) + warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case VersionFuture: - warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) + warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case VersionUnknown: warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) } - log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) + log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) } h.Warnings = warnings @@ -396,7 +410,7 @@ func (h *Hub) localSync() error { func (i *Item) setVersionState(path string, inhub bool) error { var err error - i.LocalHash, err = getSHA256(path) + i.State.LocalHash, err = getSHA256(path) if err != nil { return fmt.Errorf("failed to get sha256 of %s: %w", path, err) } @@ -412,44 +426,44 @@ func (i *Item) setVersionState(path string, inhub bool) error { return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err) } - i.LocalVersion = "?" + i.State.LocalVersion = "?" for _, version := range versions { - if i.Versions[version].Digest == i.LocalHash { - i.LocalVersion = version + if i.Versions[version].Digest == i.State.LocalHash { + i.State.LocalVersion = version break } } - if i.LocalVersion == "?" { + if i.State.LocalVersion == "?" { log.Tracef("got tainted match for %s: %s", i.Name, path) if !inhub { - i.LocalPath = path - i.Installed = true + i.State.LocalPath = path + i.State.Installed = true } - i.UpToDate = false - i.Tainted = true + i.State.UpToDate = false + i.State.Tainted = true return nil } // we got an exact match, update struct - i.Downloaded = true + i.State.Downloaded = true if !inhub { - log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.LocalVersion, i.Version) - i.LocalPath = path - i.Tainted = false + log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version) + i.State.LocalPath = path + i.State.Tainted = false // if we're walking the hub, present file doesn't means installed file - i.Installed = true + i.State.Installed = true } - if i.LocalVersion == i.Version { + if i.State.LocalVersion == i.Version { log.Tracef("%s is up-to-date", i.Name) - i.UpToDate = true + i.State.UpToDate = true } return nil diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 82107c6fe3a..717c876eda7 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -409,7 +409,7 @@ func (t *HubTestItem) InstallHub() error { // install data for parsers if needed ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } @@ -421,7 +421,7 @@ func (t *HubTestItem) InstallHub() error { // install data for scenarios if needed ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } @@ -433,7 +433,7 @@ func (t *HubTestItem) InstallHub() error { // install data for postoverflows if needed ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 0e84d9547f1..d4d9d7d0e0f 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -243,8 +243,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []str bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name) } if hubItem != nil { - bucketFactory.ScenarioVersion = hubItem.LocalVersion - bucketFactory.hash = hubItem.LocalHash + bucketFactory.ScenarioVersion = hubItem.State.LocalVersion + bucketFactory.hash = hubItem.State.LocalHash } else { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 02dfb435a9e..617e46189f3 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -67,9 +67,9 @@ func NewParsers(hub *cwhub.Hub) *Parsers { for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { for _, hubParserItem := range hub.GetItemMap(itemType) { - if hubParserItem.Installed { + if hubParserItem.State.Installed { stagefile := Stagefile{ - Filename: hubParserItem.LocalPath, + Filename: hubParserItem.State.LocalPath, Stage: hubParserItem.Stage, } if itemType == cwhub.PARSERS { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index ca40beea090..1a795e625d3 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -57,6 +57,19 @@ teardown() { assert_json '["1.10",false,false]' } +@test "do not unmarshal state attributes" { + new_hub=$( \ + jq <"$INDEX_PATH" \ + '. * {parsers:{"crowdsecurity/syslog-logs":{"tainted":true, "installed":true, "local":true}}}' + ) + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli parsers inspect crowdsecurity/syslog-logs --no-metrics + assert_output --partial 'tainted: false' + assert_output --partial 'installed: false' + assert_output --partial 'local: false' +} + @test "hub index with invalid (non semver) version numbers" { rune -0 cscli collections remove crowdsecurity/sshd --purge From e4b92af78c130391a955ef4f5c6f9b2291406c31 Mon Sep 17 00:00:00 2001 From: bui Date: Tue, 21 Nov 2023 15:24:51 +0100 Subject: [PATCH 193/263] support dedicated waap rules testing in cscli hubtest --- cmd/crowdsec-cli/hubtest.go | 162 ++++++++++++++++++++++-------------- pkg/hubtest/hubtest.go | 61 ++++++++++---- pkg/hubtest/hubtest_item.go | 22 ++--- 3 files changed, 154 insertions(+), 91 deletions(-) diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index d1fc3830c91..75b4308a8d7 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -19,6 +19,9 @@ import ( ) var HubTest hubtest.HubTest +var HubWaapTests hubtest.HubTest +var hubPtr *hubtest.HubTest +var isWaapTest bool func NewHubTestCmd() *cobra.Command { var hubPath string @@ -33,11 +36,20 @@ func NewHubTestCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error - HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath) + HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false) if err != nil { return fmt.Errorf("unable to load hubtest: %+v", err) } + HubWaapTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true) + if err != nil { + return fmt.Errorf("unable to load waap specific hubtest: %+v", err) + } + /*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to WAAP tests*/ + hubPtr = &HubTest + if isWaapTest { + hubPtr = &HubWaapTests + } return nil }, } @@ -45,6 +57,7 @@ func NewHubTestCmd() *cobra.Command { cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") + cmdHubTest.PersistentFlags().BoolVar(&isWaapTest, "waap", false, "Command relates to WAAP tests") cmdHubTest.AddCommand(NewHubTestCreateCmd()) cmdHubTest.AddCommand(NewHubTestRunCmd()) @@ -76,7 +89,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { testName := args[0] - testPath := filepath.Join(HubTest.HubTestPath, testName) + testPath := filepath.Join(hubPtr.HubTestPath, testName) if _, err := os.Stat(testPath); os.IsExist(err) { return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath) } @@ -89,53 +102,76 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return fmt.Errorf("unable to create folder '%s': %+v", testPath, err) } - // create empty log file - logFileName := fmt.Sprintf("%s.log", testName) - logFilePath := filepath.Join(testPath, logFileName) - logFile, err := os.Create(logFilePath) - if err != nil { - return err - } - logFile.Close() + configFilePath := filepath.Join(testPath, "config.yaml") - // create empty parser assertion file - parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName) - parserAssertFile, err := os.Create(parserAssertFilePath) - if err != nil { - return err - } - parserAssertFile.Close() + configFileData := &hubtest.HubTestItemConfig{} + if logType == "waap" { + //create empty nuclei template file + nucleiFileName := fmt.Sprintf("%s.yaml", testName) + nucleiFilePath := filepath.Join(testPath, nucleiFileName) + nucleiFile, err := os.Create(nucleiFilePath) + if err != nil { + return err + } + nucleiFile.Close() + configFileData.WaapRules = []string{"your_rule_here.yaml"} + configFileData.NucleiTemplate = nucleiFileName + fmt.Println() + fmt.Printf(" Test name : %s\n", testName) + fmt.Printf(" Test path : %s\n", testPath) + fmt.Printf(" Nuclei Template : %s\n", nucleiFileName) + } else { + // create empty log file + logFileName := fmt.Sprintf("%s.log", testName) + logFilePath := filepath.Join(testPath, logFileName) + logFile, err := os.Create(logFilePath) + if err != nil { + return err + } + logFile.Close() - // create empty scenario assertion file - scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName) - scenarioAssertFile, err := os.Create(scenarioAssertFilePath) - if err != nil { - return err - } - scenarioAssertFile.Close() + // create empty parser assertion file + parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName) + parserAssertFile, err := os.Create(parserAssertFilePath) + if err != nil { + return err + } + parserAssertFile.Close() + // create empty scenario assertion file + scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName) + scenarioAssertFile, err := os.Create(scenarioAssertFilePath) + if err != nil { + return err + } + scenarioAssertFile.Close() - parsers = append(parsers, "crowdsecurity/syslog-logs") - parsers = append(parsers, "crowdsecurity/dateparse-enrich") + parsers = append(parsers, "crowdsecurity/syslog-logs") + parsers = append(parsers, "crowdsecurity/dateparse-enrich") - if len(scenarios) == 0 { - scenarios = append(scenarios, "") - } + if len(scenarios) == 0 { + scenarios = append(scenarios, "") + } - if len(postoverflows) == 0 { - postoverflows = append(postoverflows, "") - } + if len(postoverflows) == 0 { + postoverflows = append(postoverflows, "") + } + configFileData.Parsers = parsers + configFileData.Scenarios = scenarios + configFileData.PostOverflows = postoverflows + configFileData.LogFile = logFileName + configFileData.LogType = logType + configFileData.IgnoreParsers = ignoreParsers + configFileData.Labels = labels + fmt.Println() + fmt.Printf(" Test name : %s\n", testName) + fmt.Printf(" Test path : %s\n", testPath) + fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath) + fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath) + fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath) + fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath) - configFileData := &hubtest.HubTestItemConfig{ - Parsers: parsers, - Scenarios: scenarios, - PostOverflows: postoverflows, - LogFile: logFileName, - LogType: logType, - IgnoreParsers: ignoreParsers, - Labels: labels, } - configFilePath := filepath.Join(testPath, "config.yaml") fd, err := os.Create(configFilePath) if err != nil { return fmt.Errorf("open: %s", err) @@ -151,14 +187,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios if err := fd.Close(); err != nil { return fmt.Errorf("close: %s", err) } - fmt.Println() - fmt.Printf(" Test name : %s\n", testName) - fmt.Printf(" Test path : %s\n", testPath) - fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath) - fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath) - fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath) - fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath) - return nil }, } @@ -188,12 +216,12 @@ func NewHubTestRunCmd() *cobra.Command { } if runAll { - if err := HubTest.LoadAllTests(); err != nil { + if err := hubPtr.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %+v", err) } } else { for _, testName := range args { - _, err := HubTest.LoadTestItem(testName) + _, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } @@ -202,8 +230,7 @@ func NewHubTestRunCmd() *cobra.Command { // set timezone to avoid DST issues os.Setenv("TZ", "UTC") - - for _, test := range HubTest.Tests { + for _, test := range hubPtr.Tests { if csConfig.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) } @@ -218,8 +245,8 @@ func NewHubTestRunCmd() *cobra.Command { PersistentPostRunE: func(cmd *cobra.Command, args []string) error { success := true testResult := make(map[string]bool) - for _, test := range HubTest.Tests { - if test.AutoGen { + for _, test := range hubPtr.Tests { + if test.AutoGen && !isWaapTest { if test.ParserAssert.AutoGenAssert { log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File) fmt.Println() @@ -341,7 +368,7 @@ func NewHubTestCleanCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } @@ -364,17 +391,23 @@ func NewHubTestInfoCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } fmt.Println() fmt.Printf(" Test name : %s\n", test.Name) fmt.Printf(" Test path : %s\n", test.Path) - fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile)) - fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName)) - fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName)) + if isWaapTest { + fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate) + fmt.Printf(" Waap Rules : %s\n", strings.Join(test.Config.WaapRules, ", ")) + } else { + fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile)) + fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName)) + fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName)) + } fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml")) } @@ -391,15 +424,15 @@ func NewHubTestListCmd() *cobra.Command { Short: "list", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := HubTest.LoadAllTests(); err != nil { + if err := hubPtr.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %s", err) } switch csConfig.Cscli.Output { case "human": - hubTestListTable(color.Output, HubTest.Tests) + hubTestListTable(color.Output, hubPtr.Tests) case "json": - j, err := json.MarshalIndent(HubTest.Tests, " ", " ") + j, err := json.MarshalIndent(hubPtr.Tests, " ", " ") if err != nil { return err } @@ -425,6 +458,7 @@ func NewHubTestCoverageCmd() *cobra.Command { Short: "coverage", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + //for this one we explictely don't do for Waap if err := HubTest.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %+v", err) } @@ -529,7 +563,7 @@ func NewHubTestEvalCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("can't load test: %+v", err) } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 0574d16db1a..fa25282b3ec 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -14,7 +14,8 @@ type HubTest struct { CrowdSecPath string CscliPath string HubPath string - HubTestPath string + HubTestPath string //generic parser/scenario tests .tests + HubWaapTestPath string //dir specific to waap tests .waap-tests HubIndexFile string TemplateConfigPath string TemplateProfilePath string @@ -33,7 +34,7 @@ const ( templateWaapProfilePath = "template_waap-profile.yaml" ) -func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { +func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isWaapTest bool) (HubTest, error) { hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) @@ -43,9 +44,6 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, if _, err = os.Stat(hubPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) } - - HubTestPath := filepath.Join(hubPath, "./.tests/") - // we can't use hubtest without crowdsec binary if _, err = exec.LookPath(crowdsecPath); err != nil { if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) { @@ -60,6 +58,39 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } } + if isWaapTest { + HubTestPath := filepath.Join(hubPath, "./.waap-tests/") + hubIndexFile := filepath.Join(hubPath, ".index.json") + + local := &csconfig.LocalHubCfg{ + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, + InstallDataDir: HubTestPath, + } + + hub, err := cwhub.NewHub(local, nil, false) + if err != nil { + return HubTest{}, fmt.Errorf("unable to load hub: %s", err) + } + + return HubTest{ + CrowdSecPath: crowdsecPath, + CscliPath: cscliPath, + HubPath: hubPath, + HubTestPath: HubTestPath, + HubIndexFile: hubIndexFile, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), + TemplateWaapProfilePath: filepath.Join(HubTestPath, templateWaapProfilePath), + TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), + HubIndex: hub, + }, nil + } + + HubTestPath := filepath.Join(hubPath, "./.tests/") + hubIndexFile := filepath.Join(hubPath, ".index.json") local := &csconfig.LocalHubCfg{ @@ -75,17 +106,15 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } return HubTest{ - CrowdSecPath: crowdsecPath, - CscliPath: cscliPath, - HubPath: hubPath, - HubTestPath: HubTestPath, - HubIndexFile: hubIndexFile, - TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), - TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), - TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), - TemplateWaapProfilePath: filepath.Join(HubTestPath, templateWaapProfilePath), - TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), - HubIndex: hub, + CrowdSecPath: crowdsecPath, + CscliPath: cscliPath, + HubPath: hubPath, + HubTestPath: HubTestPath, + HubIndexFile: hubIndexFile, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), + HubIndex: hub, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index b1a6c30ad69..9583bffecf0 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -19,17 +19,17 @@ import ( ) type HubTestItemConfig struct { - Parsers []string `yaml:"parsers"` - Scenarios []string `yaml:"scenarios"` - PostOverflows []string `yaml:"postoverflows"` - WaapRules []string `yaml:"waap-rules"` - NucleiTemplate string `yaml:"nuclei_template"` - ExpectedNucleiFailure bool `yaml:"expect_failure"` - LogFile string `yaml:"log_file"` - LogType string `yaml:"log_type"` - Labels map[string]string `yaml:"labels"` - IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + Parsers []string `yaml:"parsers,omitempty"` + Scenarios []string `yaml:"scenarios,omitempty"` + PostOverflows []string `yaml:"postoverflows,omitempty"` + WaapRules []string `yaml:"waap-rules,omitempty"` + NucleiTemplate string `yaml:"nuclei_template,omitempty"` + ExpectedNucleiFailure bool `yaml:"expect_failure,omitempty"` + LogFile string `yaml:"log_file,omitempty"` + LogType string `yaml:"log_type,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + IgnoreParsers bool `yaml:"ignore_parsers,omitempty"` // if we test a scenario, we don't want to assert on Parser + OverrideStatics []parser.ExtraField `yaml:"override_statics,omitempty"` //Allow to override statics. Executed before s00 } type HubTestItem struct { From 2c652ef92fedb2a331e56cba3cf6371339ccb81a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:43:10 +0100 Subject: [PATCH 194/263] pkg/cwhub documentation (#2607) * pkg/cwhub: package documentation * Don't repeat local state in "cscli... inspect" * lint * use proper name of the hub item instead of the filename for local items * hub update: avoid reporting local items as tainted --- cmd/crowdsec-cli/itemcommands.go | 2 +- pkg/cwhub/cwhub.go | 4 -- pkg/cwhub/cwhub_test.go | 6 +- pkg/cwhub/dataset.go | 5 +- pkg/cwhub/doc.go | 113 +++++++++++++++++++++++++++++ pkg/cwhub/enable.go | 18 ++--- pkg/cwhub/errors.go | 2 +- pkg/cwhub/helpers.go | 18 ++--- pkg/cwhub/helpers_test.go | 6 +- pkg/cwhub/hub.go | 17 +++-- pkg/cwhub/items.go | 120 +++++++++++++++---------------- pkg/cwhub/leakybucket.go | 4 +- pkg/cwhub/remote.go | 6 +- pkg/cwhub/sync.go | 62 +++++++++++----- test/bats/20_hub_items.bats | 40 +++++++++++ 15 files changed, 302 insertions(+), 121 deletions(-) create mode 100644 pkg/cwhub/doc.go diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index f36f52a48ca..82fe113c17b 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -268,7 +268,7 @@ func NewItemsInstallCmd(typeName string) *cobra.Command { func istalledParentNames(item *cwhub.Item) []string { ret := make([]string, 0) - for _, parent := range item.ParentCollections() { + for _, parent := range item.AncestorCollections() { if parent.State.Installed { ret = append(ret, parent.Name) } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 85569c64e3d..ff34bed593e 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -1,7 +1,3 @@ -// Package cwhub is responsible for installing and upgrading the local hub files. -// -// This includes retrieving the index, the items to install (parsers, scenarios, data files...) -// and managing the dependencies and taints. package cwhub import ( diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 3a260cb9a75..270f003c38f 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -27,7 +27,7 @@ const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" var responseByPath map[string]string -// testHub initializes a temporary hub with an empty json file, optionally updating it +// testHub initializes a temporary hub with an empty json file, optionally updating it. func testHub(t *testing.T, update bool) *Hub { tmpDir, err := os.MkdirTemp("", "testhub") require.NoError(t, err) @@ -67,7 +67,7 @@ func testHub(t *testing.T, update bool) *Hub { return hub } -// envSetup initializes the temporary hub and mocks the http client +// envSetup initializes the temporary hub and mocks the http client. func envSetup(t *testing.T) *Hub { setResponseByPath() log.SetLevel(log.DebugLevel) @@ -92,7 +92,7 @@ func newMockTransport() http.RoundTripper { return &mockTransport{} } -// Implement http.RoundTripper +// Implement http.RoundTripper. func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { // Create mocked http.Response response := &http.Response{ diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 79eb91573b0..e624436c869 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -13,11 +13,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) +// The DataSet is a list of data sources required by an item (built from the data: section in the yaml). type DataSet struct { Data []types.DataSource `yaml:"data,omitempty"` } -// downloadFile downloads a file and writes it to disk, with no hash verification +// downloadFile downloads a file and writes it to disk, with no hash verification. func downloadFile(url string, destPath string) error { log.Debugf("downloading %s in %s", url, destPath) @@ -50,7 +51,7 @@ func downloadFile(url string, destPath string) error { return nil } -// downloadDataSet downloads all the data files for an item +// downloadDataSet downloads all the data files for an item. func downloadDataSet(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go new file mode 100644 index 00000000000..85767265048 --- /dev/null +++ b/pkg/cwhub/doc.go @@ -0,0 +1,113 @@ +// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec. +// +// # Definitions +// +// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items. +// - The hub INDEX is a JSON file that contains a tree of available hub items. +// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version. +// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items. +// +// Once downloaded, hub items can be installed by linking to them from the configuration directory. +// If an item is present in the configuration directory but it's not a link to the local hub, it is +// considered as a LOCAL ITEM and won't be removed or upgraded. +// +// # Directory Structure +// +// A typical directory layout is the following: +// +// For the local hub (HubDir = /etc/crowdsec/hub): +// +// - /etc/crowdsec/hub/.index.json +// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml +// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml +// +// For the configuration directory (InstallDir = /etc/crowdsec): +// +// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml +// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml +// - /etc/crowdsec/scenarios/local-scenario.yaml +// +// Note that installed items are not grouped by author, this may change in the future if we want to +// support items with the same name from different authors. +// +// Only parsers and postoverflows have the concept of stage. +// +// Additionally, an item can reference a DATA SET that is installed in a different location than +// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data). +// +// - /var/lib/crowdsec/data/http_path_traversal.txt +// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt +// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt +// - /var/lib/crowdsec/data/sensitive_data.txt +// +// +// # Using the package +// +// The main entry point is the Hub struct. You can create a new instance with NewHub(). +// This constructor takes three parameters, but only the LOCAL HUB configuration is required: +// +// import ( +// "fmt" +// "github.com/crowdsecurity/crowdsec/pkg/csconfig" +// "github.com/crowdsecurity/crowdsec/pkg/cwhub" +// ) +// +// localHub := csconfig.LocalHubCfg{ +// HubIndexFile: "/etc/crowdsec/hub/.index.json", +// HubDir: "/etc/crowdsec/hub", +// InstallDir: "/etc/crowdsec", +// InstallDataDir: "/var/lib/crowdsec/data", +// } +// hub, err := cwhub.NewHub(localHub, nil, false) +// if err != nil { +// return fmt.Errorf("unable to initialize hub: %w", err) +// } +// +// Now you can use the hub to access the existing items: +// +// // list all the parsers +// for _, parser := range hub.GetItemMap(cwhub.PARSERS) { +// fmt.Printf("parser: %s\n", parser.Name) +// } +// +// // retrieve a specific collection +// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux") +// if coll == nil { +// return fmt.Errorf("collection not found") +// } +// +// You can also install items if they have already been downloaded: +// +// // install a parser +// force := false +// downloadOnly := false +// err := parser.Install(force, downloadOnly) +// if err != nil { +// return fmt.Errorf("unable to install parser: %w", err) +// } +// +// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash +// does not correspond to the latest version available in the index), a download will be attempted and you'll +// get the error "remote hub configuration is not provided". +// +// To provide the remote hub configuration, use the second parameter of NewHub(): +// +// remoteHub := cwhub.RemoteHubCfg{ +// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", +// Branch: "master", +// IndexPath: ".index.json", +// } +// updateIndex := false +// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex) +// if err != nil { +// return fmt.Errorf("unable to initialize hub: %w", err) +// } +// +// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two +// placeholders: the branch and the file path (it will be an index or an item). +// +// Setting the third parameter to true will download the latest version of the index, if available on the +// specified branch. +// There is no exported method to update the index once the hub struct is created. +// +package cwhub diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index 2a17fc7243a..d7f8b8b7148 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -11,8 +11,8 @@ import ( ) // installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local -// (eg. /etc/crowdsec/collections/xyz.yaml) -// raises an error if the path goes outside of the install dir +// (eg. /etc/crowdsec/collections/xyz.yaml). +// Raises an error if the path goes outside of the install dir. func (i *Item) installPath() (string, error) { p := i.Type if i.Stage != "" { @@ -23,8 +23,8 @@ func (i *Item) installPath() (string, error) { } // downloadPath returns the location of the actual config file in the hub -// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml) -// raises an error if the path goes outside of the hub dir +// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml). +// Raises an error if the path goes outside of the hub dir. func (i *Item) downloadPath() (string, error) { ret, err := safePath(i.hub.local.HubDir, i.RemotePath) if err != nil { @@ -34,7 +34,7 @@ func (i *Item) downloadPath() (string, error) { return ret, nil } -// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir +// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir. func (i *Item) createInstallLink() error { dest, err := i.installPath() if err != nil { @@ -63,7 +63,7 @@ func (i *Item) createInstallLink() error { return nil } -// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items +// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items. func (i *Item) enable() error { if i.State.Installed { if i.State.Tainted { @@ -97,7 +97,7 @@ func (i *Item) enable() error { return nil } -// purge removes the actual config file that was downloaded +// purge removes the actual config file that was downloaded. func (i *Item) purge() error { if !i.State.Downloaded { log.Infof("removing %s: not downloaded -- no need to remove", i.Name) @@ -124,7 +124,7 @@ func (i *Item) purge() error { return nil } -// removeInstallLink removes the symlink to the downloaded content +// removeInstallLink removes the symlink to the downloaded content. func (i *Item) removeInstallLink() error { syml, err := i.installPath() if err != nil { @@ -166,7 +166,7 @@ func (i *Item) removeInstallLink() error { return nil } -// disable removes the install link, and optionally the downloaded content +// disable removes the install link, and optionally the downloaded content. func (i *Item) disable(purge bool, force bool) error { // XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not err := i.removeInstallLink() diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go index ec389921c7b..789c2eced7b 100644 --- a/pkg/cwhub/errors.go +++ b/pkg/cwhub/errors.go @@ -7,10 +7,10 @@ import ( var ( // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. - // All attempts to download index or items will return this error. ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") ) +// IndexNotFoundError is returned when the remote hub index is not found. type IndexNotFoundError struct { URL string Branch string diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 4a03a12d964..335864050d6 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -19,7 +19,7 @@ import ( "slices" ) -// Install installs the item from the hub, downloading it if needed +// Install installs the item from the hub, downloading it if needed. func (i *Item) Install(force bool, downloadOnly bool) error { if downloadOnly && i.State.Downloaded && i.State.UpToDate { log.Infof("%s is already downloaded and up-to-date", i.Name) @@ -52,7 +52,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error { return nil } -// allDependencies returns a list of all (direct or indirect) dependencies of the item +// allDependencies returns a list of all (direct or indirect) dependencies of the item. func (i *Item) allDependencies() ([]*Item, error) { var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error @@ -94,7 +94,7 @@ func (i *Item) allDependencies() ([]*Item, error) { return ret, nil } -// Remove disables the item, optionally removing the downloaded content +// Remove disables the item, optionally removing the downloaded content. func (i *Item) Remove(purge bool, force bool) (bool, error) { if i.IsLocal() { return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) @@ -123,7 +123,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { // if the sub depends on a collection that is not a direct or indirect dependency // of the current item, it is not removed - for _, subParent := range sub.ParentCollections() { + for _, subParent := range sub.AncestorCollections() { if !purge && !subParent.State.Installed { continue } @@ -156,7 +156,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { return removed, nil } -// Upgrade downloads and applies the last version from the hub +// Upgrade downloads and applies the last version of the item from the hub. func (i *Item) Upgrade(force bool) (bool, error) { updated := false @@ -203,7 +203,7 @@ func (i *Item) Upgrade(force bool) (bool, error) { return updated, nil } -// downloadLatest downloads the latest version of the item to the hub directory +// downloadLatest downloads the latest version of the item to the hub directory. func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { // XXX: should return the path of the downloaded file (taken from download()) log.Debugf("Downloading %s %s", i.Type, i.Name) @@ -253,7 +253,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { return ret, nil } -// fetch downloads the item from the hub, verifies the hash and returns the content +// fetch downloads the item from the hub, verifies the hash and returns the content. func (i *Item) fetch() ([]byte, error) { url, err := i.hub.remote.urlTo(i.RemotePath) if err != nil { @@ -291,7 +291,7 @@ func (i *Item) fetch() ([]byte, error) { return body, nil } -// download downloads the item from the hub and writes it to the hub directory +// download downloads the item from the hub and writes it to the hub directory. func (i *Item) download(overwrite bool) (string, error) { // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { @@ -348,7 +348,7 @@ func (i *Item) download(overwrite bool) (string, error) { return finalPath, nil } -// DownloadDataIfNeeded downloads the data files for the item +// DownloadDataIfNeeded downloads the data set for the item. func (i *Item) DownloadDataIfNeeded(force bool) error { itemFilePath, err := i.installPath() if err != nil { diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 58d1a56cc82..9f8f8c1d751 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -8,8 +8,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection -// We expect the new scenario to be installed +// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection. +// We expect the new scenario to be installed. func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub := envSetup(t) @@ -110,7 +110,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) } -// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test +// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { hub, err := NewHub(local, remote, false) require.NoError(t, err, "failed to load hub index") diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 01f7eef0426..ff1c3cf15da 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -13,19 +13,22 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) +// Hub is the main structure for the package. type Hub struct { - Items HubItems + Items HubItems // Items read from HubDir and InstallDir local *csconfig.LocalHubCfg remote *RemoteHubCfg - Warnings []string + Warnings []string // Warnings encountered during sync } +// GetDataDir returns the data directory, where data sets are installed. func (h *Hub) GetDataDir() string { return h.local.InstallDataDir } -// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state -// It also downloads the index if updateIndex is true +// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state. +// If updateIndex is true, the local index file is updated from the remote before reading the state of the items. +// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) { if local == nil { return nil, fmt.Errorf("no hub configuration found") @@ -55,7 +58,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) return hub, nil } -// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections +// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections. func (h *Hub) parseIndex() error { bidx, err := os.ReadFile(h.local.HubIndexFile) if err != nil { @@ -91,7 +94,7 @@ func (h *Hub) parseIndex() error { return nil } -// ItemStats returns total counts of the hub items +// ItemStats returns total counts of the hub items, including local and tainted. func (h *Hub) ItemStats() []string { loaded := "" local := 0 @@ -131,7 +134,7 @@ func (h *Hub) ItemStats() []string { return ret } -// updateIndex downloads the latest version of the index and writes it to disk if it changed +// updateIndex downloads the latest version of the index and writes it to disk if it changed. func (h *Hub) updateIndex() error { body, err := h.remote.fetchIndex() if err != nil { diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 7e0a10f4c3a..1c82adb6fd4 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -12,7 +12,7 @@ import ( ) const ( - // managed item types + // managed item types. COLLECTIONS = "collections" PARSERS = "parsers" POSTOVERFLOWS = "postoverflows" @@ -20,59 +20,57 @@ const ( ) const ( - VersionUpToDate = iota - VersionUpdateAvailable - VersionUnknown - VersionFuture + versionUpToDate = iota // the latest version from index is installed + versionUpdateAvailable // not installed, or lower than latest + versionUnknown // local file with no version, or invalid version number + versionFuture // local version is higher latest, but is included in the index: should not happen ) -// The order is important, as it is used to range over sub-items in collections -var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} +var ( + // The order is important, as it is used to range over sub-items in collections. + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} +) type HubItems map[string]map[string]*Item // ItemVersion is used to detect the version of a given item // by comparing the hash of each version to the local file. -// If the item does not match any known version, it is considered tainted. +// If the item does not match any known version, it is considered tainted (modified). type ItemVersion struct { Digest string `json:"digest,omitempty"` // meow Deprecated bool `json:"deprecated,omitempty"` } -// ItemState is used to keep the local state (i.e. at runtime) of an item -// This data is not stored in the index, but is displayed in the output of "cscli ... inspect" +// ItemState is used to keep the local state (i.e. at runtime) of an item. +// This data is not stored in the index, but is displayed with "cscli ... inspect". type ItemState struct { - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow + LocalHash string `json:"local_hash,omitempty"` Installed bool `json:"installed"` Downloaded bool `json:"downloaded"` UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` // has it been locally modified? - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any + Tainted bool `json:"tainted"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` } -// Item represents an object managed in the hub. It can be a parser, scenario, collection.. +// Item is created from an index file and enriched with local info. type Item struct { - // back pointer to the hub, to retrieve subitems and call install/remove methods - hub *Hub - - // local (deployed) info - State ItemState - - // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json - Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json - - // remote (hub) info - RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml - Version string `json:"version,omitempty"` // the last version - Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions + hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods + + State ItemState `json:"-" yaml:"-"` // local state, not stored in the index + + Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // usually "author/name" + FileName string `json:"file_name,omitempty"` // eg. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Author string `json:"author,omitempty"` + References []string `json:"references,omitempty" yaml:"references,omitempty"` + + RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml + Version string `json:"version,omitempty"` // the last available version + Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` @@ -81,17 +79,18 @@ type Item struct { Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` } +// HasSubItems returns true if items of this type can have sub-items. Currently only collections. func (i *Item) HasSubItems() bool { return i.Type == COLLECTIONS } +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). func (i *Item) IsLocal() bool { return i.State.Installed && !i.State.Downloaded } -// MarshalJSON is used to add the "local" field to the json output -// (i.e. with cscli ... inspect -o json) -// It must not use a pointer receiver +// MarshalJSON is used to prepare the output for "cscli ... inspect -o json". +// It must not use a pointer receiver. func (i Item) MarshalJSON() ([]byte, error) { type Alias Item @@ -121,9 +120,8 @@ func (i Item) MarshalJSON() ([]byte, error) { }) } -// MarshalYAML is used to add the "local" field to the yaml output -// (i.e. with cscli ... inspect -o raw) -// It must not use a pointer receiver +// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw". +// It must not use a pointer receiver. func (i Item) MarshalYAML() (interface{}, error) { type Alias Item @@ -138,7 +136,7 @@ func (i Item) MarshalYAML() (interface{}, error) { }, nil } -// SubItems returns a slice of sub-item pointers, excluding the ones that were not found +// SubItems returns a slice of sub-items, excluding the ones that were not found. func (i *Item) SubItems() []*Item { sub := make([]*Item, 0) @@ -211,8 +209,8 @@ func (i *Item) logMissingSubItems() { } } -// ParentCollections returns the list of items (collections) that have this item as a direct dependency -func (i *Item) ParentCollections() []*Item { +// AncestorCollections returns a slice of items (collections) that have this item as a direct or indirect dependency. +func (i *Item) AncestorCollections() []*Item { ret := make([]*Item, 0) for _, parentName := range i.State.BelongsToCollections { @@ -228,7 +226,7 @@ func (i *Item) ParentCollections() []*Item { } // Status returns the status of the item as a string and an emoji -// ie. "enabled,update-available" and emoji.Warning +// (eg. "enabled,update-available" and emoji.Warning). func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false @@ -269,47 +267,47 @@ func (i *Item) Status() (string, emoji.Emoji) { return status, emo } -// versionStatus: semver requires 'v' prefix +// versionStatus returns the status of the item version compared to the hub version. +// semver requires the 'v' prefix. func (i *Item) versionStatus() int { local, err := semver.NewVersion(i.State.LocalVersion) if err != nil { - return VersionUnknown + return versionUnknown } // hub versions are already validated while syncing, ignore errors latest, _ := semver.NewVersion(i.Version) if local.LessThan(latest) { - return VersionUpdateAvailable + return versionUpdateAvailable } if local.Equal(latest) { - return VersionUpToDate + return versionUpToDate } - return VersionFuture + return versionFuture } -// validPath returns true if the (relative) path is allowed for the item -// dirNname: the directory name (ie. crowdsecurity) -// fileName: the filename (ie. apache2-logs.yaml) +// validPath returns true if the (relative) path is allowed for the item. +// dirNname: the directory name (ie. crowdsecurity). +// fileName: the filename (ie. apache2-logs.yaml). func (i *Item) validPath(dirName, fileName string) bool { return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") } -// GetItemMap returns the map of items for a given type +// GetItemMap returns the map of items for a given type. func (h *Hub) GetItemMap(itemType string) map[string]*Item { return h.Items[itemType] } -// GetItem returns the item from hub based on its type and full name (author/name) +// GetItem returns an item from hub based on its type and full name (author/name). func (h *Hub) GetItem(itemType string, itemName string) *Item { return h.GetItemMap(itemType)[itemName] } -// GetItemNames returns the list of (full) item names for a given type -// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx -// The names can be used to retrieve the item with GetItem() +// GetItemNames returns a slice of (full) item names for a given type +// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx). func (h *Hub) GetItemNames(itemType string) []string { m := h.GetItemMap(itemType) if m == nil { @@ -324,7 +322,7 @@ func (h *Hub) GetItemNames(itemType string) []string { return names } -// GetAllItems returns a slice of all the items, installed or not +// GetAllItems returns a slice of all the items of a given type, installed or not. func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { items, ok := h.Items[itemType] if !ok { @@ -343,7 +341,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) { return ret, nil } -// GetInstalledItems returns the list of installed items +// GetInstalledItems returns a slice of the installed items of a given type. func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { items, ok := h.Items[itemType] if !ok { @@ -361,7 +359,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { return retItems, nil } -// GetInstalledItemNames returns the names of the installed items +// GetInstalledItemNames returns the names of the installed items of a given type. func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) { items, err := h.GetInstalledItems(itemType) if err != nil { @@ -377,7 +375,7 @@ func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) { return retStr, nil } -// SortItemSlice sorts a slice of items by name, case insensitive +// SortItemSlice sorts a slice of items by name, case insensitive. func SortItemSlice(items []*Item) { sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) diff --git a/pkg/cwhub/leakybucket.go b/pkg/cwhub/leakybucket.go index 5d97153cbf8..8143e9433ee 100644 --- a/pkg/cwhub/leakybucket.go +++ b/pkg/cwhub/leakybucket.go @@ -10,7 +10,7 @@ import ( "strings" ) -// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary. func itemKey(itemPath string) (string, error) { f, err := os.Lstat(itemPath) if err != nil { @@ -37,7 +37,7 @@ func itemKey(itemPath string) (string, error) { return fmt.Sprintf("%s/%s", author, fname), nil } -// GetItemByPath retrieves the item from the hub index based on its path. +// GetItemByPath retrieves an item from the hub index based on its local path. func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { itemKey, err := itemKey(itemPath) if err != nil { diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index c98dfa8f5c2..c1eb5a7080f 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -6,14 +6,14 @@ import ( "net/http" ) -// RemoteHubCfg contains where to find the remote hub, which branch etc. +// RemoteHubCfg is used to retrieve index and items from the remote hub. type RemoteHubCfg struct { Branch string URLTemplate string IndexPath string } -// urlTo builds the URL to download a file from the remote hub +// urlTo builds the URL to download a file from the remote hub. func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { if r == nil { return "", ErrNilRemoteHub @@ -27,7 +27,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil } -// fetchIndex downloads the index from the hub and returns the content +// fetchIndex downloads the index from the hub and returns the content. func (r *RemoteHubCfg) fetchIndex() ([]byte, error) { if r == nil { return nil, ErrNilRemoteHub diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 4325017b4a0..d5d7410151c 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -12,13 +12,14 @@ import ( "github.com/Masterminds/semver/v3" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" ) func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } -// linkTarget returns the target of a symlink, or empty string if it's dangling +// linkTarget returns the target of a symlink, or empty string if it's dangling. func linkTarget(path string) (string, error) { hubpath, err := os.Readlink(path) if err != nil { @@ -52,7 +53,7 @@ func getSHA256(filepath string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -// information used to create a new Item, from a file path +// information used to create a new Item, from a file path. type itemFileInfo struct { inhub bool fname string @@ -127,7 +128,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { return ret, nil } -// sortedVersions returns the input data, sorted in reverse order (new, old) by semver +// sortedVersions returns the input data, sorted in reverse order (new, old) by semver. func sortedVersions(raw []string) ([]string, error) { vs := make([]*semver.Version, len(raw)) @@ -150,10 +151,14 @@ func sortedVersions(raw []string) ([]string, error) { return ret, nil } -func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item { +func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { + type localItemName struct { + Name string `yaml:"name"` + } + _, fileName := filepath.Split(path) - return &Item{ + item := &Item{ hub: h, Name: info.fname, Stage: info.stage, @@ -165,6 +170,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item { UpToDate: true, }, } + + // try to read the name from the file + itemName := localItemName{} + + itemContent, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", path, err) + } + + err = yaml.Unmarshal(itemContent, &itemName) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err) + } + + if itemName.Name != "" { + item.Name = itemName.Name + } + + return item, nil } func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { @@ -198,7 +222,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { if !info.inhub { log.Tracef("%s is a local file, skip", path) - h.Items[info.ftype][info.fname] = newLocalItem(h, path, info) + item, err := newLocalItem(h, path, info) + if err != nil { + return err + } + h.Items[info.ftype][item.Name] = item return nil } @@ -269,13 +297,13 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { return nil } -// checkSubItems checks for the presence, taint and version state of sub-items +// checkSubItems checks for the presence, taint and version state of sub-items. func (h *Hub) checkSubItems(v *Item) error { if !v.HasSubItems() { return nil } - if v.versionStatus() != VersionUpToDate { + if v.versionStatus() != versionUpToDate { log.Debugf("%s dependencies not checked: not up-to-date", v.Name) return nil } @@ -321,7 +349,7 @@ func (h *Hub) checkSubItems(v *Item) error { return nil } -// syncDir scans a directory for items, and updates the Hub state accordingly +// syncDir scans a directory for items, and updates the Hub state accordingly. func (h *Hub) syncDir(dir string) error { // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { @@ -347,7 +375,7 @@ func (h *Hub) syncDir(dir string) error { return nil } -// insert a string in a sorted slice, case insensitive, and return the new slice +// insert a string in a sorted slice, case insensitive, and return the new slice. func insertInOrderNoCase(sl []string, value string) []string { i := sort.Search(len(sl), func(i int) bool { return strings.ToLower(sl[i]) >= strings.ToLower(value) @@ -356,7 +384,7 @@ func insertInOrderNoCase(sl []string, value string) []string { return append(sl[:i], append([]string{value}, sl[i:]...)...) } -// localSync updates the hub state with downloaded, installed and local items +// localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { err := h.syncDir(h.local.InstallDir) if err != nil { @@ -387,16 +415,18 @@ func (h *Hub) localSync() error { vs := item.versionStatus() switch vs { - case VersionUpToDate: // latest + case versionUpToDate: // latest if err := h.checkSubItems(item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) } - case VersionUpdateAvailable: // not up-to-date + case versionUpdateAvailable: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) - case VersionFuture: + case versionFuture: warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) - case VersionUnknown: - warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) + case versionUnknown: + if !item.IsLocal() { + warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) + } } log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 1a795e625d3..73f2bbcb9f7 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -107,3 +107,43 @@ teardown() { refute_output refute_stderr } + +@test "a local item is not tainted" { + # not from cscli... inspect + rune -0 mkdir -p "$CONFIG_DIR/collections" + rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections inspect foobar.yaml -o json + rune -0 jq -e '.tainted==false' <(output) + + rune -0 cscli collections install crowdsecurity/sshd + rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.tainted==true' <(output) + + # and not from hub update + rune -0 cscli hub update + assert_stderr --partial "collection crowdsecurity/sshd is tainted" + refute_stderr --partial "collection foobar.yaml is tainted" +} + +@test "a local item's name defaults to its filename" { + rune -0 mkdir -p "$CONFIG_DIR/collections" + rune -0 touch "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections list -o json + rune -0 jq -r '.[][].name' <(output) + assert_output "foobar.yaml" + rune -0 cscli collections list foobar.yaml + rune -0 cscli collections inspect foobar.yaml -o json + rune -0 jq -e '.installed==true' <(output) +} + +@test "a local item can provide its own name" { + rune -0 mkdir -p "$CONFIG_DIR/collections" + echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml" + rune -0 cscli collections list -o json + rune -0 jq -r '.[][].name' <(output) + assert_output "hi-its-me" + rune -0 cscli collections list hi-its-me + rune -0 cscli collections inspect hi-its-me -o json + rune -0 jq -e '.installed==true' <(output) +} From ef9b6acbf85571786f250708e56efaeccd0579c7 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 22 Nov 2023 10:54:48 +0100 Subject: [PATCH 195/263] use generic implem for cscli waap-configs --- cmd/crowdsec-cli/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 7c6f3576b2b..848cfbb220d 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -242,12 +242,12 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - rootCmd.AddCommand(NewWaapRulesCmd()) // Keep it like this for now, we'll switch later to the generic implementation - rootCmd.AddCommand(NewWaapConfigsCmd()) // Keep it like this for now, we'll switch later to the generic implementation + rootCmd.AddCommand(NewWaapRulesCmd()) // Keep it like this for now, we'll switch later to the generic implementation rootCmd.AddCommand(NewItemsCmd("collections")) rootCmd.AddCommand(NewItemsCmd("parsers")) rootCmd.AddCommand(NewItemsCmd("scenarios")) rootCmd.AddCommand(NewItemsCmd("postoverflows")) + rootCmd.AddCommand(NewItemsCmd("waap-configs")) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) From 56c616f70dc75a04dadd4e50c4f610469d9a630c Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 22 Nov 2023 15:00:15 +0100 Subject: [PATCH 196/263] delete cscli/waap_configs.go --- cmd/crowdsec-cli/waap_configs.go | 151 ------------------------------- 1 file changed, 151 deletions(-) delete mode 100644 cmd/crowdsec-cli/waap_configs.go diff --git a/cmd/crowdsec-cli/waap_configs.go b/cmd/crowdsec-cli/waap_configs.go deleted file mode 100644 index cfe6be74671..00000000000 --- a/cmd/crowdsec-cli/waap_configs.go +++ /dev/null @@ -1,151 +0,0 @@ -package main - -import ( - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewWaapConfigsCmd() *cobra.Command { - cmdWaapConfigs := &cobra.Command{ - Use: "waap-configs [waap-config]...", - Short: "Manage hub waap configs", - Example: `cscli waap-configs list -a -cscli waap-configs install crowdsecurity/crs -cscli waap-configs inspect crowdsecurity/crs -cscli waap-configs upgrade crowdsecurity/crs -cscli waap-configs remove crowdsecurity/crs -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"waap-config"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig, require.RemoteHub(csConfig)); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdWaapConfigs.AddCommand(NewCmdWaapConfigsInstall()) - cmdWaapConfigs.AddCommand(NewCmdWaapConfigsRemove()) - cmdWaapConfigs.AddCommand(NewCmdWaapConfigsUpgrade()) - cmdWaapConfigs.AddCommand(NewCmdWaapConfigsInspect()) - cmdWaapConfigs.AddCommand(NewCmdWaapConfigsList()) - - return cmdWaapConfigs -} - -func NewCmdWaapConfigsInstall() *cobra.Command { - cmdWaapConfigsInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given waap config(s)", - Long: `Fetch and install one or more waap configs from the hub`, - Example: `cscli waap-configs install crowdsecurity/vpatch`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.WAAP_CONFIGS, args, toComplete) - }, - RunE: itemsInstallRunner(hubItemTypes[cwhub.WAAP_CONFIGS]), - } - - flags := cmdWaapConfigsInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple waap rules") - - return cmdWaapConfigsInstall -} - -func NewCmdWaapConfigsRemove() *cobra.Command { - cmdWaapConfigsRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given waap config(s)", - Long: `remove one or more waap configs`, - Example: `cscli waap-configs remove crowdsecurity/vpatch`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) - }, - RunE: itemsRemoveRunner(hubItemTypes[cwhub.WAAP_CONFIGS]), - } - - flags := cmdWaapConfigsRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the waap configs") - - return cmdWaapConfigsRemove -} - -func NewCmdWaapConfigsUpgrade() *cobra.Command { - cmdWaapConfigsUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given waap config(s)", - Long: `Fetch and upgrade one or more waap configs from the hub`, - Example: `cscli waap-configs upgrade crowdsecurity/crs`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) - }, - RunE: itemsUpgradeRunner(hubItemTypes[cwhub.WAAP_CONFIGS]), - } - - flags := cmdWaapConfigsUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the waap configs") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdWaapConfigsUpgrade -} - -func NewCmdWaapConfigsInspect() *cobra.Command { - //FIXME: we need to show the "compiled" rules - it := hubItemTypes[cwhub.WAAP_CONFIGS] - cmdWaapConfigsInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a waap config", - Long: `Inspect a waap config`, - Example: `cscli waap-configs inspect crowdsecurity/vpatch`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_CONFIGS, args, toComplete) - }, - RunE: itemsInspectRunner(it), - } - - flags := cmdWaapConfigsInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdWaapConfigsInspect -} - -func NewCmdWaapConfigsList() *cobra.Command { - cmdWaapConfigsList := &cobra.Command{ - Use: "list [waap-config]...", - Short: "List waap configs", - Long: `List of installed/available/specified waap configs`, - Example: `cscli waap-configs list -cscli waap-configs list -a -cscli waap-configs list crowdsecurity/crs`, - DisableAutoGenTag: true, - RunE: itemsListRunner(hubItemTypes[cwhub.WAAP_CONFIGS]), - } - - flags := cmdWaapConfigsList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdWaapConfigsList -} From dd6e539717c0a47c0f9119a1de452becbed7c9e3 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 22 Nov 2023 15:40:57 +0100 Subject: [PATCH 197/263] fix hubtest coverage and some opti --- cmd/crowdsec-cli/hubtest.go | 38 +++++++++++++++++- cmd/crowdsec-cli/hubtest_table.go | 20 ++++++++++ pkg/hubtest/coverage.go | 65 +++++++++++++++++++++++++++++-- pkg/hubtest/hubtest_item.go | 35 ++++++++--------- pkg/hubtest/nucleirunner.go | 35 ++++++++++------- pkg/hubtest/utils.go | 20 ++++++++++ 6 files changed, 175 insertions(+), 38 deletions(-) diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 75b4308a8d7..5ff15af7095 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -452,6 +452,7 @@ func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool var showOnlyPercent bool + var showWaapCov bool var cmdHubTestCoverage = &cobra.Command{ Use: "coverage", @@ -465,11 +466,13 @@ func NewHubTestCoverageCmd() *cobra.Command { var err error scenarioCoverage := []hubtest.Coverage{} parserCoverage := []hubtest.Coverage{} + waapRuleCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 + waapRuleCoveragePercent := 0 // if both are false (flag by default), show both - showAll := !showScenarioCov && !showParserCov + showAll := !showScenarioCov && !showParserCov && !showWaapCov if showParserCov || showAll { parserCoverage, err = HubTest.GetParsersCoverage() @@ -501,13 +504,30 @@ func NewHubTestCoverageCmd() *cobra.Command { scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } + if showWaapCov || showAll { + waapRuleCoverage, err = HubTest.GetWaapCoverage() + if err != nil { + return fmt.Errorf("while getting scenario coverage: %s", err) + } + + waapRuleTested := 0 + for _, test := range waapRuleCoverage { + if test.TestsCount > 0 { + waapRuleTested++ + } + } + waapRuleCoveragePercent = int(math.Round((float64(waapRuleTested) / float64(len(waapRuleCoverage)) * 100))) + } + if showOnlyPercent { if showAll { - fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent) + fmt.Printf("parsers=%d%%\nscenarios=%d%%\nwaap_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, waapRuleCoveragePercent) } else if showParserCov { fmt.Printf("parsers=%d%%", parserCoveragePercent) } else if showScenarioCov { fmt.Printf("scenarios=%d%%", scenarioCoveragePercent) + } else if showWaapCov { + fmt.Printf("waap_rules=%d%%", waapRuleCoveragePercent) } os.Exit(0) } @@ -521,6 +541,11 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { hubTestScenarioCoverageTable(color.Output, scenarioCoverage) } + + if showWaapCov || showAll { + hubTestWaapRuleCoverageTable(color.Output, waapRuleCoverage) + } + fmt.Println() if showParserCov || showAll { fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent) @@ -528,6 +553,9 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } + if showWaapCov || showAll { + fmt.Printf("WAAP RULES : %d%% of coverage\n", waapRuleCoveragePercent) + } case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") if err != nil { @@ -539,6 +567,11 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) + dump, err = json.MarshalIndent(waapRuleCoverage, "", " ") + if err != nil { + return err + } + fmt.Printf("%s", dump) default: return fmt.Errorf("only human/json output modes are supported") } @@ -550,6 +583,7 @@ func NewHubTestCoverageCmd() *cobra.Command { cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") + cmdHubTestCoverage.PersistentFlags().BoolVar(&showWaapCov, "waap", false, "Show only waap coverage") return cmdHubTestCoverage } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 9b31a79a20f..4a24d8d1715 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -61,6 +61,26 @@ func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t.Render() } +func hubTestWaapRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) { + t := newLightTable(out) + t.SetHeaders("Waap Rule", "Status", "Number of tests") + t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) + t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) + + parserTested := 0 + + for _, test := range coverage { + status := emoji.RedCircle.String() + if test.TestsCount > 0 { + status = emoji.GreenCircle.String() + parserTested++ + } + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + } + + t.Render() +} + func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Scenario", "Status", "Number of tests") diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 5d3b79fe11e..6381d850490 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -7,9 +7,10 @@ import ( "path/filepath" "strings" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type Coverage struct { @@ -18,6 +19,65 @@ type Coverage struct { PresentIn map[string]bool //poorman's set } +func (h *HubTest) GetWaapCoverage() ([]Coverage, error) { + if _, ok := h.HubIndex.Items[cwhub.WAAP_RULES]; !ok { + return nil, fmt.Errorf("no waap rules in hub index") + } + + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.WAAP_RULES]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, + TestsCount: 0, + PresentIn: make(map[string]bool), + } + } + + // parser the expressions a-la-oneagain + waapTestConfigs, err := filepath.Glob(".waap-tests/*/config.yaml") + if err != nil { + return nil, fmt.Errorf("while find waap-tests config: %s", err) + } + + for _, waapTestConfigPath := range waapTestConfigs { + configFileData := &HubTestItemConfig{} + yamlFile, err := os.ReadFile(waapTestConfigPath) + if err != nil { + log.Printf("unable to open waap test config file '%s': %s", waapTestConfigPath, err) + continue + } + err = yaml.Unmarshal(yamlFile, configFileData) + if err != nil { + return nil, fmt.Errorf("unmarshal: %v", err) + } + + for _, waapRulesFile := range configFileData.WaapRules { + waapRuleData := &waap_rule.CustomRule{} + yamlFile, err := os.ReadFile(waapRulesFile) + if err != nil { + log.Printf("unable to open waap rule '%s': %s", waapRulesFile, err) + } + err = yaml.Unmarshal(yamlFile, waapRuleData) + if err != nil { + return nil, fmt.Errorf("unmarshal: %v", err) + } + waapRuleName := waapRuleData.Name + + for idx, cov := range coverage { + if cov.Name == waapRuleName { + coverage[idx].TestsCount++ + coverage[idx].PresentIn[waapTestConfigPath] = true + } + } + } + } + + return coverage, nil +} + func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { return nil, fmt.Errorf("no parsers in hub index") @@ -127,7 +187,6 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { return nil, fmt.Errorf("while find scenario asserts : %s", err) } - for _, assert := range passerts { file, err := os.Open(assert) if err != nil { diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 2fca53c12ef..5fb700dd798 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -3,12 +3,11 @@ package hubtest import ( "errors" "fmt" - "net" + "net/url" "os" "os/exec" "path/filepath" "strings" - "time" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -84,6 +83,11 @@ const ( ScenarioResultFileName = "bucket-dump.yaml" BucketPourResultFileName = "bucketpour-dump.yaml" + + TestBouncerApiKey = "this_is_a_bad_password" + + DefaultNucleiTarget = "http://127.0.0.1:80/" + DefaultWaapHost = "127.0.0.1:4241" ) func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { @@ -558,7 +562,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } //hardcode bouncer key - cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "waaptests", "-k", "this_is_a_bad_password"} + cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "waaptests", "-k", TestBouncerApiKey} cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) output, err = cscliBouncerCmd.CombinedOutput() @@ -576,25 +580,20 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { crowdsecDaemon.Start() //wait for the waap port to be available + if _, err := IsAlive(DefaultWaapHost); err != nil { + return fmt.Errorf("Waap is down: %s", err) + } - start := time.Now() - - for { - conn, err := net.Dial("tcp", "127.0.0.1:4241") - if err == nil { - log.Debugf("waap is up after %s", time.Since(start)) - conn.Close() - break - } - time.Sleep(500 * time.Millisecond) - if time.Since(start) > 10*time.Second { - log.Fatalf("took more than 10s for waap to be available, abort") - } + // check if the target is available + nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget) + nucleiTargetHost := nucleiTargetParsedURL.Host + if _, err := IsAlive(nucleiTargetHost); err != nil { + return fmt.Errorf("Target is down: %s", err) } nucleiConfig := NucleiConfig{ Path: "nuclei", - OutputDir: testPath, + OutputDir: t.RuntimePath, CmdLineOptions: []string{"-ev", //allow variables from environment "-nc", //no colors in output "-dresp", //dump response @@ -602,7 +601,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { }, } - err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, "http://127.0.0.1:80/") + err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget) if t.Config.ExpectedNucleiFailure { if err != nil && errors.Is(err, NucleiTemplateFail) { diff --git a/pkg/hubtest/nucleirunner.go b/pkg/hubtest/nucleirunner.go index 048070a216f..e3c6af73ca1 100644 --- a/pkg/hubtest/nucleirunner.go +++ b/pkg/hubtest/nucleirunner.go @@ -20,38 +20,43 @@ type NucleiConfig struct { var NucleiTemplateFail = errors.New("Nuclei template failed") -func (ts *NucleiConfig) RunNucleiTemplate(test_name string, template_path string, target string) error { +func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error { tstamp := time.Now().Unix() - //template_path is the full path to the template, we just want the name ie. "sqli-random-test" - tmp := strings.Split(template_path, "/") + //templatePath is the full path to the template, we just want the name ie. "sqli-random-test" + tmp := strings.Split(templatePath, "/") template := strings.Split(tmp[len(tmp)-1], ".")[0] - output_prefix := fmt.Sprintf("%s/%s_%s-%d", ts.OutputDir, test_name, template, tstamp) + outputPrefix := fmt.Sprintf("%s/%s_%s-%d", nc.OutputDir, testName, template, tstamp) args := []string{ "-u", target, - "-t", template_path, - "-o", output_prefix + ".json", + "-t", templatePath, + "-o", outputPrefix + ".json", } - args = append(args, ts.CmdLineOptions...) - cmd := exec.Command(ts.Path, args...) + args = append(args, nc.CmdLineOptions...) + cmd := exec.Command(nc.Path, args...) var out bytes.Buffer - var out_err bytes.Buffer + var outErr bytes.Buffer + cmd.Stdout = &out - cmd.Stderr = &out_err + cmd.Stderr = &outErr + err := cmd.Run() - if err := os.WriteFile(output_prefix+"_stdout.txt", out.Bytes(), 0644); err != nil { + + if err := os.WriteFile(outputPrefix+"_stdout.txt", out.Bytes(), 0644); err != nil { log.Warningf("Error writing stdout: %s", err) } - if err := os.WriteFile(output_prefix+"_stderr.txt", out_err.Bytes(), 0644); err != nil { + + if err := os.WriteFile(outputPrefix+"_stderr.txt", outErr.Bytes(), 0644); err != nil { log.Warningf("Error writing stderr: %s", err) } + if err != nil { log.Warningf("Error running nuclei: %s", err) - log.Warningf("Stdout saved to %s", output_prefix+"_stdout.txt") - log.Warningf("Stderr saved to %s", output_prefix+"_stderr.txt") - log.Warningf("Nuclei generated output saved to %s", output_prefix+".json") + log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt") + log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt") + log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json") return err } else if len(out.String()) == 0 { //No stdout means no finding, it means our test failed diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 090f1f85e3d..b264b787064 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -2,9 +2,13 @@ package hubtest import ( "fmt" + "net" "os" "path/filepath" "sort" + "time" + + log "github.com/sirupsen/logrus" ) func sortedMapKeys[V any](m map[string]V) []string { @@ -106,3 +110,19 @@ func CopyDir(src string, dest string) error { return nil } + +func IsAlive(target string) (bool, error) { + start := time.Now() + for { + conn, err := net.Dial("tcp", target) + if err == nil { + log.Debugf("waap is up after %s", time.Since(start)) + conn.Close() + return true, nil + } + time.Sleep(500 * time.Millisecond) + if time.Since(start) > 10*time.Second { + return false, fmt.Errorf("took more than 10s for %s to be available", target) + } + } +} From b6899e0c101742009724f050e47ab129792dc60b Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 22 Nov 2023 16:25:20 +0100 Subject: [PATCH 198/263] add more debug when unauthorized --- pkg/acquisition/modules/waap/waap.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 5e0174cea59..de9e8f29f2e 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -331,6 +331,9 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { if !exists || time.Now().After(expiration) { if !w.IsAuth(apiKey) { rw.WriteHeader(http.StatusUnauthorized) + clientIP := r.Header.Get(waf.IPHeaderName) + remoteIP := r.RemoteAddr + w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) return } From 710d8a438a5b16593ce0536f855161f04e61fe32 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Wed, 22 Nov 2023 16:27:22 +0100 Subject: [PATCH 199/263] oups --- pkg/acquisition/modules/waap/waap.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index de9e8f29f2e..982b5471323 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -322,7 +322,10 @@ func (w *WaapSource) IsAuth(apiKey string) bool { // should this be in the runner ? func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { apiKey := r.Header.Get(waf.APIKeyHeaderName) + clientIP := r.Header.Get(waf.IPHeaderName) + remoteIP := r.RemoteAddr if apiKey == "" { + w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) rw.WriteHeader(http.StatusUnauthorized) return } @@ -331,8 +334,6 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { if !exists || time.Now().After(expiration) { if !w.IsAuth(apiKey) { rw.WriteHeader(http.StatusUnauthorized) - clientIP := r.Header.Get(waf.IPHeaderName) - remoteIP := r.RemoteAddr w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) return } From 118da5b4233a86b922cd483d009e8288b849d914 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 22 Nov 2023 18:08:16 +0100 Subject: [PATCH 200/263] up --- cmd/crowdsec-cli/item_metrics.go | 2 +- cmd/crowdsec-cli/waap_rules.go | 45 +++++++++++++++++++++++++++++++- pkg/waf/waap_rule/types.go | 4 +++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index ad64fb1daa9..fb138141b8f 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -33,7 +33,7 @@ func ShowMetrics(hubItem *cwhub.Item) error { } } case cwhub.WAAP_RULES: - log.Fatalf("FIXME: not implemented yet") + log.Error("FIXME: not implemented yet") default: // no metrics for this item type } diff --git a/cmd/crowdsec-cli/waap_rules.go b/cmd/crowdsec-cli/waap_rules.go index 40c80bc7ebc..342ff63eed2 100644 --- a/cmd/crowdsec-cli/waap_rules.go +++ b/cmd/crowdsec-cli/waap_rules.go @@ -1,11 +1,19 @@ package main import ( + "fmt" + "os" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/text/cases" + "golang.org/x/text/language" + "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/waf" + "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" ) func NewWaapRulesCmd() *cobra.Command { @@ -109,6 +117,41 @@ func NewCmdWaapRulesUpgrade() *cobra.Command { return cmdWaapRulesUpgrade } +func WaapRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + f := itemsInspectRunner(hubItemTypes[cwhub.WAAP_RULES]) + if err := f(cmd, args); err != nil { + return err + } + if csConfig.Cscli.Output == "human" { + hub, _ := require.Hub(csConfig, nil) + for _, name := range args { + hubItem := hub.GetItem(itemType.name, name) + waapRule := waf.WaapCollectionConfig{} + yamlContent, err := os.ReadFile(hubItem.State.LocalPath) + if err != nil { + return fmt.Errorf("unable to read file %s : %s", hubItem.State.LocalPath, err) + } + if err := yaml.Unmarshal(yamlContent, &waapRule); err != nil { + return fmt.Errorf("unable to unmarshal yaml file %s : %s", hubItem.State.LocalPath, err) + } + + for _, ruleType := range waap_rule.SupportedTypes() { + fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) + for _, rule := range waapRule.Rules { + convertedRule, _, err := rule.Convert(ruleType, waapRule.Name) + if err != nil { + return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) + } + fmt.Println(convertedRule) + } + } + } + } + return nil + } +} + func NewCmdWaapRulesInspect() *cobra.Command { //FIXME; show the "compiled" rule cmdWaapRulesInspect := &cobra.Command{ @@ -121,7 +164,7 @@ func NewCmdWaapRulesInspect() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, - RunE: itemsInspectRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: WaapRulesInspectRunner(hubItemTypes[cwhub.WAAP_RULES]), } flags := cmdWaapRulesInspect.Flags() diff --git a/pkg/waf/waap_rule/types.go b/pkg/waf/waap_rule/types.go index 80ac560a0fd..ba7bfdafbd6 100644 --- a/pkg/waf/waap_rule/types.go +++ b/pkg/waf/waap_rule/types.go @@ -3,3 +3,7 @@ package waap_rule const ( ModsecurityRuleType = "modsecurity" ) + +func SupportedTypes() []string { + return []string{ModsecurityRuleType} +} From f77d9e043a99894bc6f9eb90ec9345ca5593adf7 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 23 Nov 2023 14:51:05 +0100 Subject: [PATCH 201/263] up --- pkg/waf/waap.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index c30275a7587..25aef729ad7 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -238,7 +238,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { } func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { - for _, rule := range w.CompiledOnMatch { + for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, ParsedRequest{})) if err != nil { @@ -286,14 +286,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, map[string]interface{}{ - // "req": request, - // "RemoveInbandRuleByID": w.RemoveInbandRuleByID, - // "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, - // "SetAction": response.SetAction, - // "SetRemediationByID": response.SetRemediationByID, - // "CancelEvent": response.CancelEvent, - }) + _, err := expr.Run(applyExpr, GetHookEnv(w, request)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue From 946fbbb8a2d57c2538cd835350429d1a7227dc94 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 24 Nov 2023 15:57:49 +0100 Subject: [PATCH 202/263] up --- pkg/waf/env.go | 44 ------------------- pkg/waf/tx.go | 26 ++++++++++++ pkg/waf/waap.go | 94 ++++++++++++++++++++++++++++++++--------- pkg/waf/waf_expr_lib.go | 6 +++ pkg/waf/waf_helpers.go | 34 +++++++++++++++ 5 files changed, 139 insertions(+), 65 deletions(-) delete mode 100644 pkg/waf/env.go create mode 100644 pkg/waf/tx.go diff --git a/pkg/waf/env.go b/pkg/waf/env.go deleted file mode 100644 index 2322a6e43af..00000000000 --- a/pkg/waf/env.go +++ /dev/null @@ -1,44 +0,0 @@ -package waf - -import ( - "github.com/crowdsecurity/coraza/v3" - "github.com/crowdsecurity/coraza/v3/experimental" -) - -type ExtendedTransaction struct { - Tx experimental.FullTransaction -} - -func NewExtendedTransaction(engine coraza.WAF, uuid string) ExtendedTransaction { - inBoundTx := engine.NewTransactionWithID(uuid) - expTx := inBoundTx.(experimental.FullTransaction) - tx := NewTransaction(expTx) - return tx -} - -func NewTransaction(tx experimental.FullTransaction) ExtendedTransaction { - return ExtendedTransaction{Tx: tx} -} - -func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { - t.Tx.RemoveRuleByID(id) - return nil -} - -// simply used to ease the compilation & runtime of the hooks -func GetHookEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { - return map[string]interface{}{ - "inband_rules": w.InBandRules, - "outband_rules": w.OutOfBandRules, - "req": request, - "RemoveInbandRuleByID": w.RemoveInbandRuleByID, - "RemoveOutbandRuleByID": w.RemoveOutbandRuleByID, - "SetAction": w.SetAction, - "SetActionByTag": w.SetActionByTag, - "SetHTTPCode": w.SetHTTPCode, - "SetActionByID": w.SetActionByID, - "CancelEvent": w.CancelEvent, - "IsInBand": request.IsInBand, - "IsOutBand": request.IsOutBand, - } -} diff --git a/pkg/waf/tx.go b/pkg/waf/tx.go new file mode 100644 index 00000000000..416ad953ef9 --- /dev/null +++ b/pkg/waf/tx.go @@ -0,0 +1,26 @@ +package waf + +import ( + "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/experimental" +) + +type ExtendedTransaction struct { + Tx experimental.FullTransaction +} + +func NewExtendedTransaction(engine coraza.WAF, uuid string) ExtendedTransaction { + inBoundTx := engine.NewTransactionWithID(uuid) + expTx := inBoundTx.(experimental.FullTransaction) + tx := NewTransaction(expTx) + return tx +} + +func NewTransaction(tx experimental.FullTransaction) ExtendedTransaction { + return ExtendedTransaction{Tx: tx} +} + +func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { + t.Tx.RemoveRuleByID(id) + return nil +} diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 25aef729ad7..44992050808 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -47,6 +47,7 @@ type WaapTempResponse struct { Action string //allow, deny, captcha, log HTTPResponseCode int SendEvent bool //do we send an internal event on rule match + SendAlert bool //do we send an alert on rule match } type WaapSubEngineOpts struct { @@ -240,7 +241,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, ParsedRequest{})) + output, err := expr.Run(rule.FilterExpr, GetOnLoadEnv(w)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -256,7 +257,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetHookEnv(w, ParsedRequest{})) + _, err := expr.Run(applyExpr, GetOnLoadEnv(w)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue @@ -270,7 +271,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, request)) + output, err := expr.Run(rule.FilterExpr, GetOnMatchEnv(w, request)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -286,7 +287,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetHookEnv(w, request)) + _, err := expr.Run(applyExpr, GetOnMatchEnv(w, request)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue @@ -299,7 +300,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetHookEnv(w, request)) + output, err := expr.Run(rule.FilterExpr, GetPreEvalEnv(w, request)) if err != nil { return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) } @@ -316,7 +317,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetHookEnv(w, request)) + _, err := expr.Run(applyExpr, GetPreEvalEnv(w, request)) if err != nil { log.Errorf("unable to apply filter: %s", err) continue @@ -335,31 +336,80 @@ add the helpers to: */ -func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { - return w.InBandTx.RemoveRuleByIDWithError(id) +// func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { +func (w *WaapRuntimeConfig) RemoveInbandRuleByID(params ...any) (any, error) { + id := params[0].(int) + _ = w.InBandTx.RemoveRuleByIDWithError(id) + return nil, nil } -func (w *WaapRuntimeConfig) CancelEvent() error { +func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { w.Response.SendEvent = false - return nil + return nil, nil } -func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { +// func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { +func (w *WaapRuntimeConfig) DisableInBandRuleByID(params ...any) (any, error) { panic("not implemented") - return nil + return nil, nil } -func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { +// func (w *WaapRuntimeConfig) DisableInBandRuleByTag(id int) error { +func (w *WaapRuntimeConfig) DisableInBandRuleByTag(params ...any) (any, error) { panic("not implemented") - return nil + return nil, nil +} + +// func (w *WaapRuntimeConfig) DisableOutBandRuleByID(tag string) error { +func (w *WaapRuntimeConfig) DisableOutBandRuleByID(params ...any) (any, error) { + panic("not implemented") + return nil, nil +} + +// func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { +func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(params ...any) (any, error) { + panic("not implemented") + return nil, nil +} + +func (w *WaapRuntimeConfig) SendEvent(params ...any) (any, error) { + w.Response.SendEvent = true + return nil, nil } -func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { - return w.OutOfBandTx.RemoveRuleByIDWithError(id) +func (w *WaapRuntimeConfig) SendAlert(params ...any) (any, error) { + w.Response.SendAlert = true + return nil, nil } -func (w *WaapRuntimeConfig) SetAction(action string) error { +func (w *WaapRuntimeConfig) CancelAlert(params ...any) (any, error) { + w.Response.SendAlert = false + return nil, nil +} + +// func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { +func (w *WaapRuntimeConfig) SetActionByTag(params ...any) (any, error) { + panic("not implemented") + return nil, nil +} + +// func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { +func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { + panic("not implemented") + return nil, nil +} + +// func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { +func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { + id := params[0].(int) + _ = w.OutOfBandTx.RemoveRuleByIDWithError(id) + return nil, nil +} + +// func (w *WaapRuntimeConfig) SetAction(action string) error { +func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { //log.Infof("setting to %s", action) + action := params[0].(string) switch action { case "allow": w.Response.Action = action @@ -375,15 +425,17 @@ func (w *WaapRuntimeConfig) SetAction(action string) error { w.Response.Action = action w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode default: - return fmt.Errorf("unknown action %s", action) + return nil, fmt.Errorf("unknown action %s", action) } - return nil + return nil, nil } -func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { +// func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { +func (w *WaapRuntimeConfig) SetHTTPCode(params ...any) (any, error) { + code := params[0].(int) w.Response.HTTPResponseCode = code - return nil + return nil, nil } type BodyResponse struct { diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 33dd1f4e4ee..48685957de4 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -8,6 +8,12 @@ type exprCustomFunc struct { signature []interface{} } +var onLoadExprFuncs = []exprCustomFunc{} + +var preEvalExprFuncs = []exprCustomFunc{} + +var onMatchExprFuncs = []exprCustomFunc{} + var exprFuncs = []exprCustomFunc{ /*{ name: "SetRulesToInband", diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 97b1f30f0c3..22a35a8b9bb 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -30,3 +30,37 @@ func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { } return baseHelpers } + +func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { + return map[string]interface{}{ + "DisableInBandRuleByID": w.DisableInBandRuleByID, + "DisableOutBandRuleByID": w.DisableOutBandRuleByID, + "DisableInBandRuleByTag": w.DisableInBandRuleByTag, + "DisableOutBandRuleByTag": w.DisableOutBandRuleByTag, + } +} + +func GetPreEvalEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { + return map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "RemoveInBandRuleByID": w.RemoveInbandRuleByID, + "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, + "SetRemediationByTag": w.SetActionByTag, + "SetRemdiationByID": w.SetActionByID, + } +} + +func GetOnMatchEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { + return map[string]interface{}{ + "req": request, + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "SetRemediation": w.SetAction, + "SetReturnCode": w.SetHTTPCode, + "CancelEvent": w.CancelEvent, + "SendEvent": w.SendEvent, + "CancelAlert": w.CancelAlert, + "SendAlert": w.SendAlert, + } +} From b1653aea63984239ebc39db4951deb80fa7fe9a7 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 27 Nov 2023 10:43:32 +0100 Subject: [PATCH 203/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 34 +++++------ pkg/waf/request.go | 3 +- pkg/waf/tx.go | 62 +++++++++++++++++++++ pkg/waf/waap.go | 60 ++++++++++++-------- pkg/waf/waf_expr_lib.go | 21 ++++++- pkg/waf/waf_helpers.go | 7 ++- 6 files changed, 144 insertions(+), 43 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index a8f83d41e4c..27d53985d41 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -6,7 +6,6 @@ import ( "time" "github.com/crowdsecurity/coraza/v3" - "github.com/crowdsecurity/coraza/v3/experimental" corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" @@ -77,7 +76,7 @@ func (r *WaapRunner) Init(datadir string) error { return nil } -func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *waf.ParsedRequest) error { +func (r *WaapRunner) processRequest(tx waf.ExtendedTransaction, request *waf.ParsedRequest) error { var in *corazatypes.Interruption var err error request.Tx = tx @@ -92,7 +91,14 @@ func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *wa //We don't close the transaction here, as it will reset coraza internal state and break variable tracking }() - request.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) + //pre eval (expr) rules + err = r.WaapRuntime.ProcessPreEvalRules(request) + if err != nil { + r.logger.Errorf("unable to process PreEval rules: %s", err) + //FIXME: should we abort here ? + } + + request.Tx.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) for k, v := range request.Args { for _, vv := range v { @@ -151,14 +157,16 @@ func (r *WaapRunner) processRequest(tx experimental.FullTransaction, request *wa } func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { - tx := r.WaapInbandEngine.NewTransactionWithID(request.UUID) - err := r.processRequest(tx.(experimental.FullTransaction), request) + tx := waf.NewExtendedTransaction(r.WaapInbandEngine, request.UUID) + r.WaapRuntime.InBandTx = tx + err := r.processRequest(tx, request) return err } func (r *WaapRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { - tx := r.WaapOutbandEngine.NewTransactionWithID(request.UUID) - err := r.processRequest(tx.(experimental.FullTransaction), request) + tx := waf.NewExtendedTransaction(r.WaapInbandEngine, request.UUID) + r.WaapRuntime.OutOfBandTx = tx + err := r.processRequest(tx, request) return err } @@ -180,14 +188,8 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { //to measure the time spent in the WAF startParsing := time.Now() - //pre eval (expr) rules - err := r.WaapRuntime.ProcessPreEvalRules(request) - if err != nil { - r.logger.Errorf("unable to process PreEval rules: %s", err) - continue - } //inband WAAP rules - err = r.ProcessInBandRules(&request) + err := r.ProcessInBandRules(&request) if err != nil { r.logger.Errorf("unable to process InBand rules: %s", err) continue @@ -206,7 +208,7 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { r.logger.Debugf("inband rules matched : %d", in.RuleID) r.WaapRuntime.Response.InBandInterrupt = true - err = r.WaapRuntime.ProcessOnMatchRules(request) + err = r.WaapRuntime.ProcessOnMatchRules(&request) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) continue @@ -236,7 +238,7 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("outband rules matched : %d", in.RuleID) r.WaapRuntime.Response.OutOfBandInterrupt = true - err = r.WaapRuntime.ProcessOnMatchRules(request) + err = r.WaapRuntime.ProcessOnMatchRules(&request) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) continue diff --git a/pkg/waf/request.go b/pkg/waf/request.go index f7b7a11b737..904d5fdd304 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -6,7 +6,6 @@ import ( "net/http" "net/url" - "github.com/crowdsecurity/coraza/v3/experimental" "github.com/google/uuid" ) @@ -74,7 +73,7 @@ type ParsedRequest struct { Body []byte TransferEncoding []string UUID string - Tx experimental.FullTransaction + Tx ExtendedTransaction ResponseChannel chan WaapTempResponse IsInBand bool IsOutBand bool diff --git a/pkg/waf/tx.go b/pkg/waf/tx.go index 416ad953ef9..ce44aec95ec 100644 --- a/pkg/waf/tx.go +++ b/pkg/waf/tx.go @@ -3,6 +3,8 @@ package waf import ( "github.com/crowdsecurity/coraza/v3" "github.com/crowdsecurity/coraza/v3/experimental" + "github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes" + "github.com/crowdsecurity/coraza/v3/types" ) type ExtendedTransaction struct { @@ -24,3 +26,63 @@ func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { t.Tx.RemoveRuleByID(id) return nil } + +func (t *ExtendedTransaction) IsRuleEngineOff() bool { + return t.Tx.IsRuleEngineOff() +} + +func (t *ExtendedTransaction) ProcessLogging() { + t.Tx.ProcessLogging() +} + +func (t *ExtendedTransaction) ProcessConnection(client string, cPort int, server string, sPort int) { + t.Tx.ProcessConnection(client, cPort, server, sPort) +} + +func (t *ExtendedTransaction) AddGetRequestArgument(name string, value string) { + t.Tx.AddGetRequestArgument(name, value) +} + +func (t *ExtendedTransaction) ProcessURI(uri string, method string, httpVersion string) { + t.Tx.ProcessURI(uri, method, httpVersion) +} + +func (t *ExtendedTransaction) AddRequestHeader(name string, value string) { + t.Tx.AddRequestHeader(name, value) +} + +func (t *ExtendedTransaction) SetServerName(name string) { + t.Tx.SetServerName(name) +} + +func (t *ExtendedTransaction) ProcessRequestHeaders() *types.Interruption { + return t.Tx.ProcessRequestHeaders() +} + +func (t *ExtendedTransaction) ProcessRequestBody() (*types.Interruption, error) { + return t.Tx.ProcessRequestBody() +} + +func (t *ExtendedTransaction) WriteRequestBody(body []byte) (*types.Interruption, int, error) { + return t.Tx.WriteRequestBody(body) +} + +func (t *ExtendedTransaction) Interruption() *types.Interruption { + return t.Tx.Interruption() +} + +func (t *ExtendedTransaction) IsInterrupted() bool { + return t.Tx.IsInterrupted() +} + +func (t *ExtendedTransaction) Variables() plugintypes.TransactionVariables { + return t.Tx.Variables() +} + +func (t *ExtendedTransaction) MatchedRules() []types.MatchedRule { + return t.Tx.MatchedRules() +} + +func (t *ExtendedTransaction) ID() string { + return t.Tx.ID() +} diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 44992050808..b58d414c10c 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -21,18 +21,34 @@ type Hook struct { ApplyExpr []*vm.Program `yaml:"-"` } -// @tko : todo - debug mode -func (h *Hook) Build() error { +const ( + hookOnLoad = iota + hookPreEval + hookOnMatch +) +// @tko : todo - debug mode +func (h *Hook) Build(hookStage int) error { + + ctx := map[string]interface{}{} + switch hookStage { + case hookOnLoad: + ctx = GetOnLoadEnv(&WaapRuntimeConfig{}) + case hookPreEval: + ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + case hookOnMatch: + ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + } + opts := GetExprWAFOptions(ctx) if h.Filter != "" { - program, err := expr.Compile(h.Filter) //FIXME: opts + program, err := expr.Compile(h.Filter, opts...) //FIXME: opts if err != nil { return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err) } h.FilterExpr = program } for _, apply := range h.Apply { - program, err := expr.Compile(apply, GetExprWAFOptions(GetHookEnv(&WaapRuntimeConfig{}, ParsedRequest{}))...) + program, err := expr.Compile(apply, opts...) if err != nil { return fmt.Errorf("unable to compile apply %s : %w", apply, err) } @@ -204,7 +220,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { //load hooks for _, hook := range wc.OnLoad { - err := hook.Build() + err := hook.Build(hookOnLoad) if err != nil { return nil, fmt.Errorf("unable to build on_load hook : %s", err) } @@ -212,7 +228,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { } for _, hook := range wc.PreEval { - err := hook.Build() + err := hook.Build(hookPreEval) if err != nil { return nil, fmt.Errorf("unable to build pre_eval hook : %s", err) } @@ -220,7 +236,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { } for _, hook := range wc.OnMatch { - err := hook.Build() + err := hook.Build(hookOnMatch) if err != nil { return nil, fmt.Errorf("unable to build on_match hook : %s", err) } @@ -243,7 +259,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { if rule.FilterExpr != nil { output, err := expr.Run(rule.FilterExpr, GetOnLoadEnv(w)) if err != nil { - return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run waap on_load filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -259,7 +275,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, applyExpr := range rule.ApplyExpr { _, err := expr.Run(applyExpr, GetOnLoadEnv(w)) if err != nil { - log.Errorf("unable to apply filter: %s", err) + log.Errorf("unable to apply waap on_load expr: %s", err) continue } } @@ -267,13 +283,13 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { return nil } -func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { +func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { output, err := expr.Run(rule.FilterExpr, GetOnMatchEnv(w, request)) if err != nil { - return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -289,7 +305,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { for _, applyExpr := range rule.ApplyExpr { _, err := expr.Run(applyExpr, GetOnMatchEnv(w, request)) if err != nil { - log.Errorf("unable to apply filter: %s", err) + log.Errorf("unable to apply waap on_match expr: %s", err) continue } } @@ -297,12 +313,12 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request ParsedRequest) error { return nil } -func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { +func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { output, err := expr.Run(rule.FilterExpr, GetPreEvalEnv(w, request)) if err != nil { - return fmt.Errorf("unable to run filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run waap pre_eval filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -319,7 +335,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request ParsedRequest) error { for _, applyExpr := range rule.ApplyExpr { _, err := expr.Run(applyExpr, GetPreEvalEnv(w, request)) if err != nil { - log.Errorf("unable to apply filter: %s", err) + log.Errorf("unable to apply waap pre_eval expr: %s", err) continue } } @@ -343,6 +359,13 @@ func (w *WaapRuntimeConfig) RemoveInbandRuleByID(params ...any) (any, error) { return nil, nil } +// func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { +func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { + id := params[0].(int) + _ = w.OutOfBandTx.RemoveRuleByIDWithError(id) + return nil, nil +} + func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { w.Response.SendEvent = false return nil, nil @@ -399,13 +422,6 @@ func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { return nil, nil } -// func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { -func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { - id := params[0].(int) - _ = w.OutOfBandTx.RemoveRuleByIDWithError(id) - return nil, nil -} - // func (w *WaapRuntimeConfig) SetAction(action string) error { func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { //log.Infof("setting to %s", action) diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 48685957de4..23e2c711bec 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -8,7 +8,26 @@ type exprCustomFunc struct { signature []interface{} } -var onLoadExprFuncs = []exprCustomFunc{} +/* +func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { + return map[string]interface{}{ + "DisableInBandRuleByID": w.DisableInBandRuleByID, + "DisableOutBandRuleByID": w.DisableOutBandRuleByID, + "DisableInBandRuleByTag": w.DisableInBandRuleByTag, + "DisableOutBandRuleByTag": w.DisableOutBandRuleByTag, + } +} +*/ + +/*var onLoadExprFuncs = []exprCustomFunc{ + { + name: "DisableInBandRuleByID", + function: w.DisableInBandRuleByID, + signature: []interface{}{ + new(func(int) error), + }, + }, +}*/ var preEvalExprFuncs = []exprCustomFunc{} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 22a35a8b9bb..968d8decadb 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -32,6 +32,7 @@ func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { } func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { + //FIXME: use expr.Function instead of this return map[string]interface{}{ "DisableInBandRuleByID": w.DisableInBandRuleByID, "DisableOutBandRuleByID": w.DisableOutBandRuleByID, @@ -40,7 +41,8 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { } } -func GetPreEvalEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { +func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { + //FIXME: use expr.Function instead of this return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, @@ -51,7 +53,8 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]inter } } -func GetOnMatchEnv(w *WaapRuntimeConfig, request ParsedRequest) map[string]interface{} { +func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { + //FIXME: use expr.Function instead of this return map[string]interface{}{ "req": request, "IsInBand": request.IsInBand, From e7505f5b2edf0d5b1d019f50edb1cf927673c5d2 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 27 Nov 2023 13:14:40 +0100 Subject: [PATCH 204/263] up --- pkg/acquisition/modules/waap/waap_runner.go | 2 +- pkg/waf/waap.go | 23 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 27d53985d41..c1a7cecd89e 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -164,7 +164,7 @@ func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { } func (r *WaapRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { - tx := waf.NewExtendedTransaction(r.WaapInbandEngine, request.UUID) + tx := waf.NewExtendedTransaction(r.WaapOutbandEngine, request.UUID) r.WaapRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) return err diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index b58d414c10c..c72076db37b 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -92,6 +92,7 @@ type WaapRuntimeConfig struct { Response WaapTempResponse //should we store matched rules here ? + Logger *log.Entry } type WaapConfig struct { @@ -191,7 +192,7 @@ func (wc *WaapConfig) GetDataDir() string { } func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { - ret := &WaapRuntimeConfig{} + ret := &WaapRuntimeConfig{Logger: wc.Logger.WithField("component", "waap_runtime_config")} ret.Name = wc.Name ret.Config = wc ret.DefaultRemediation = wc.DefaultRemediation @@ -355,6 +356,7 @@ add the helpers to: // func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { func (w *WaapRuntimeConfig) RemoveInbandRuleByID(params ...any) (any, error) { id := params[0].(int) + w.Logger.Debugf("removing inband rule %d", id) _ = w.InBandTx.RemoveRuleByIDWithError(id) return nil, nil } @@ -362,11 +364,13 @@ func (w *WaapRuntimeConfig) RemoveInbandRuleByID(params ...any) (any, error) { // func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { id := params[0].(int) + w.Logger.Debugf("removing outband rule %d", id) _ = w.OutOfBandTx.RemoveRuleByIDWithError(id) return nil, nil } func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { + w.Logger.Debugf("canceling event") w.Response.SendEvent = false return nil, nil } @@ -396,16 +400,19 @@ func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(params ...any) (any, error) } func (w *WaapRuntimeConfig) SendEvent(params ...any) (any, error) { + w.Logger.Debugf("sending event") w.Response.SendEvent = true return nil, nil } func (w *WaapRuntimeConfig) SendAlert(params ...any) (any, error) { + w.Logger.Debugf("sending alert") w.Response.SendAlert = true return nil, nil } func (w *WaapRuntimeConfig) CancelAlert(params ...any) (any, error) { + w.Logger.Debugf("canceling alert") w.Response.SendAlert = false return nil, nil } @@ -426,6 +433,7 @@ func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { //log.Infof("setting to %s", action) action := params[0].(string) + w.Logger.Debugf("setting action to %s", action) switch action { case "allow": w.Response.Action = action @@ -450,6 +458,7 @@ func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { // func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { func (w *WaapRuntimeConfig) SetHTTPCode(params ...any) (any, error) { code := params[0].(int) + w.Logger.Debugf("setting http code to %d", code) w.Response.HTTPResponseCode = code return nil, nil } @@ -467,8 +476,16 @@ func (w *WaapRuntimeConfig) GenerateResponse(interrupted bool) BodyResponse { resp.HTTPStatus = w.Config.PassedHTTPCode return resp } - resp.Action = w.Config.DefaultRemediation - resp.HTTPStatus = w.Config.BlockedHTTPCode + resp.Action = w.Response.Action + if resp.Action == "" { + resp.Action = w.Config.DefaultRemediation + } + w.Logger.Debugf("action is %s", resp.Action) + resp.HTTPStatus = w.Response.HTTPResponseCode + if resp.HTTPStatus == 0 { + resp.HTTPStatus = w.Config.BlockedHTTPCode + } + w.Logger.Debugf("http status is %d", resp.HTTPStatus) return resp } From d8514907903a355ae485d5c4b9fa086ebf25bc15 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 27 Nov 2023 13:41:00 +0100 Subject: [PATCH 205/263] up --- pkg/acquisition/modules/waap/utils.go | 3 --- pkg/waf/waap_rules_collection.go | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index e302f868212..c904891b4d4 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -11,7 +11,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/ptr" - "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -204,8 +203,6 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() - spew.Dump(waf.WaapRulesDetails) - name := "NOT_SET" version := "NOT_SET" hash := "NOT_SET" diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index 1b579a73532..c699a724f0b 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -49,6 +49,7 @@ type RulesDetails struct { var WaapRulesDetails = make(map[int]RulesDetails) func LoadCollection(pattern string) ([]WaapCollection, error) { + //FIXME: have a proper logger here, inheriting from waap-config to have consistent log levels ret := make([]WaapCollection, 0) for _, waapRule := range waapRules { @@ -106,7 +107,7 @@ func LoadCollection(pattern string) ([]WaapCollection, error) { log.Errorf("unable to convert rule %s : %s", rule.Name, err) return nil, err } - log.Infof("Adding rule %s", strRule) + log.Debugf("Adding rule %s", strRule) waapCol.Rules = append(waapCol.Rules, strRule) //We only take the first id, as it's the one of the "main" rule From 3eb272c4e06df948b20328b0ba6e9722e7e1abfe Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 28 Nov 2023 10:15:12 +0100 Subject: [PATCH 206/263] Add metrics --- cmd/crowdsec-cli/metrics.go | 30 +++++++- cmd/crowdsec-cli/metrics_table.go | 35 ++++++++- cmd/crowdsec/metrics.go | 3 +- pkg/acquisition/modules/waap/metrics.go | 12 ++- pkg/acquisition/modules/waap/utils.go | 3 +- pkg/acquisition/modules/waap/waap.go | 10 +++ pkg/acquisition/modules/waap/waap_runner.go | 1 - pkg/waf/request.go | 82 +++++++++++++-------- 8 files changed, 134 insertions(+), 42 deletions(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index a03614aae2a..f1e347e51b8 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -63,6 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error lapi_machine_stats := map[string]map[string]map[string]int{} lapi_bouncer_stats := map[string]map[string]map[string]int{} decisions_stats := map[string]map[string]map[string]int{} + waap_engine_stats := map[string]map[string]int{} + waap_rule_stats := map[string]map[string]map[string]int{} alerts_stats := map[string]int{} stash_stats := map[string]struct { Type string @@ -226,10 +228,30 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error Type string Count int }{Type: mtype, Count: ival} + case "cs_waf_reqs_total": + if _, ok := waap_engine_stats[metric.Labels["waap_engine"]]; !ok { + waap_engine_stats[metric.Labels["waap_engine"]] = make(map[string]int, 0) + } + waap_engine_stats[metric.Labels["waap_engine"]]["processed"] = ival + case "cs_waf_block_total": + if _, ok := waap_engine_stats[metric.Labels["waap_engine"]]; !ok { + waap_engine_stats[metric.Labels["waap_engine"]] = make(map[string]int, 0) + } + waap_engine_stats[metric.Labels["waap_engine"]]["blocked"] = ival + case "cs_waf_rule_hits": + waapEngine := metric.Labels["waap_engine"] + ruleID := metric.Labels["rule_id"] + if _, ok := waap_rule_stats[waapEngine]; !ok { + waap_rule_stats[waapEngine] = make(map[string]map[string]int, 0) + } + if _, ok := waap_rule_stats[waapEngine][ruleID]; !ok { + waap_rule_stats[waapEngine][ruleID] = make(map[string]int, 0) + } + waap_rule_stats[waapEngine][ruleID]["processed"] = ival default: + log.Infof("unknown: %+v", fam.Name) continue } - } } @@ -244,6 +266,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error decisionStatsTable(out, decisions_stats) alertStatsTable(out, alerts_stats) stashStatsTable(out, stash_stats) + waapMetricsToTable(out, waap_engine_stats) + waapRulesToTable(out, waap_rule_stats) return nil } @@ -282,7 +306,6 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error var noUnit bool - func runMetrics(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -314,7 +337,6 @@ func runMetrics(cmd *cobra.Command, args []string) error { return nil } - func NewMetricsCmd() *cobra.Command { cmdMetrics := &cobra.Command{ Use: "metrics", @@ -322,7 +344,7 @@ func NewMetricsCmd() *cobra.Command { Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: runMetrics, + RunE: runMetrics, } flags := cmdMetrics.PersistentFlags() diff --git a/cmd/crowdsec-cli/metrics_table.go b/cmd/crowdsec-cli/metrics_table.go index 69706c7acf2..51606cd9ce4 100644 --- a/cmd/crowdsec-cli/metrics_table.go +++ b/cmd/crowdsec-cli/metrics_table.go @@ -90,7 +90,7 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) { keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"} if numRows, err := metricsToTable(t, stats, keys); err != nil { - log.Warningf("while collecting acquis stats: %s", err) + log.Warningf("while collecting bucket stats: %s", err) } else if numRows > 0 { renderTableTitle(out, "\nBucket Metrics:") t.Render() @@ -113,6 +113,37 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) { } } +func waapMetricsToTable(out io.Writer, metrics map[string]map[string]int) { + t := newTable(out) + t.SetRowLines(false) + t.SetHeaders("WAF Engine", "Processed", "Blocked") + t.SetAlignment(table.AlignLeft, table.AlignLeft) + keys := []string{"processed", "blocked"} + if numRows, err := metricsToTable(t, metrics, keys); err != nil { + log.Warningf("while collecting waap stats: %s", err) + } else if numRows > 0 { + renderTableTitle(out, "\nWaap Metrics:") + t.Render() + } +} + +func waapRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) { + for waapEngine, waapEngineRulesStats := range metrics { + t := newTable(out) + t.SetRowLines(false) + t.SetHeaders("Rule ID", "Processed") + t.SetAlignment(table.AlignLeft, table.AlignLeft) + keys := []string{"processed"} + if numRows, err := metricsToTable(t, waapEngineRulesStats, keys); err != nil { + log.Warningf("while collecting waap stats: %s", err) + } else if numRows > 0 { + renderTableTitle(out, fmt.Sprintf("\nWaap '%s' Rules Metrics:", waapEngine)) + t.Render() + } + } + +} + func parserStatsTable(out io.Writer, stats map[string]map[string]int) { t := newTable(out) t.SetRowLines(false) @@ -122,7 +153,7 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) { keys := []string{"hits", "parsed", "unparsed"} if numRows, err := metricsToTable(t, stats, keys); err != nil { - log.Warningf("while collecting acquis stats: %s", err) + log.Warningf("while collecting parsers stats: %s", err) } else if numRows > 0 { renderTableTitle(out, "\nParser Metrics:") t.Render() diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 42530148caf..33b36b851b5 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -164,6 +164,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsCurrentCount, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, waap.WafGlobalParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, + waap.WafBlockCounter, ) } else { log.Infof("Loading prometheus collectors") @@ -174,7 +175,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - waap.WafGlobalParsingHistogram, waap.WafInbandParsingHistogram, waap.WafOutbandParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, + waap.WafGlobalParsingHistogram, waap.WafInbandParsingHistogram, waap.WafOutbandParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, waap.WafBlockCounter, ) } diff --git a/pkg/acquisition/modules/waap/metrics.go b/pkg/acquisition/modules/waap/metrics.go index db9747250e4..3545786f01f 100644 --- a/pkg/acquisition/modules/waap/metrics.go +++ b/pkg/acquisition/modules/waap/metrics.go @@ -34,7 +34,15 @@ var WafReqCounter = prometheus.NewCounterVec( Name: "cs_waf_reqs_total", Help: "Total events processed by the WAF.", }, - []string{"source"}, + []string{"source", "waap_engine"}, +) + +var WafBlockCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_block_total", + Help: "Total events blocked by the WAF.", + }, + []string{"source", "waap_engine"}, ) var WafRuleHits = prometheus.NewCounterVec( @@ -42,5 +50,5 @@ var WafRuleHits = prometheus.NewCounterVec( Name: "cs_waf_rule_hits", Help: "Count of triggered rule, by rule_id and type (inband/outofband).", }, - []string{"rule_id", "type"}, + []string{"rule_id", "type", "waap_engine", "source"}, ) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index c904891b4d4..bd0e3b3a589 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -201,7 +201,8 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest evt.Waap.HasOutBandMatches = true } - WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() + // TODO: Fetch the Name of the rule when possible + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind, "source": req.RemoteAddrNormalized, "waap_engine": req.WaapEngine}).Inc() name := "NOT_SET" version := "NOT_SET" diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 982b5471323..a478c2779bb 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -133,6 +133,10 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { return fmt.Errorf("waap_config or waap_config_path must be set") } + if wc.config.Name == "" { + wc.config.Name = fmt.Sprintf("%s:%d%s", wc.config.ListenAddr, wc.config.ListenPort, wc.config.Path) + } + csConfig := csconfig.GetConfig() wc.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL) wc.AuthCache = NewAuthCache() @@ -349,10 +353,16 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusInternalServerError) return } + parsedRequest.WaapEngine = w.config.Name + + WafReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel + if response.InBandInterrupt { + WafBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() + } waapResponse := w.WaapRuntime.GenerateResponse(response.InBandInterrupt) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index c1a7cecd89e..9a07f0966c7 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -184,7 +184,6 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { request.IsInBand = true request.IsOutBand = false - WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() //to measure the time spent in the WAF startParsing := time.Now() diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 904d5fdd304..882a7142585 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -3,10 +3,13 @@ package waf import ( "fmt" "io" + "net" "net/http" "net/url" + "github.com/google/uuid" + log "github.com/sirupsen/logrus" ) const ( @@ -60,23 +63,25 @@ const ( // } type ParsedRequest struct { - RemoteAddr string - Host string - ClientIP string - URI string - Args url.Values - ClientHost string - Headers http.Header - URL *url.URL - Method string - Proto string - Body []byte - TransferEncoding []string - UUID string - Tx ExtendedTransaction - ResponseChannel chan WaapTempResponse - IsInBand bool - IsOutBand bool + RemoteAddr string + Host string + ClientIP string + URI string + Args url.Values + ClientHost string + Headers http.Header + URL *url.URL + Method string + Proto string + Body []byte + TransferEncoding []string + UUID string + Tx ExtendedTransaction + ResponseChannel chan WaapTempResponse + IsInBand bool + IsOutBand bool + WaapEngine string + RemoteAddrNormalized string } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine @@ -123,20 +128,35 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err) } + RemoteAddrNormalized := "" + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + log.Errorf("Invalid waap remote IP source %v: %s", r.RemoteAddr, err.Error()) + RemoteAddrNormalized = r.RemoteAddr + } else { + ip := net.ParseIP(host) + if ip == nil { + log.Errorf("Invalid waap remote IP address source %v: %s", r.RemoteAddr, err.Error()) + RemoteAddrNormalized = r.RemoteAddr + } + RemoteAddrNormalized = ip.String() + } + return ParsedRequest{ - RemoteAddr: r.RemoteAddr, - UUID: uuid.New().String(), - ClientHost: clientHost, - ClientIP: clientIP, - URI: parsedURL.Path, - Method: clientMethod, - Host: r.Host, - Headers: r.Header, - URL: r.URL, - Proto: r.Proto, - Body: body, - Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args - TransferEncoding: r.TransferEncoding, - ResponseChannel: make(chan WaapTempResponse), + RemoteAddr: r.RemoteAddr, + UUID: uuid.New().String(), + ClientHost: clientHost, + ClientIP: clientIP, + URI: parsedURL.Path, + Method: clientMethod, + Host: r.Host, + Headers: r.Header, + URL: r.URL, + Proto: r.Proto, + Body: body, + Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args + TransferEncoding: r.TransferEncoding, + ResponseChannel: make(chan WaapTempResponse), + RemoteAddrNormalized: RemoteAddrNormalized, }, nil } From 3683a7a02a2a91bc97b87237fb4f21f5d6704562 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 28 Nov 2023 11:05:29 +0100 Subject: [PATCH 207/263] up --- cmd/crowdsec-cli/metrics.go | 2 +- cmd/crowdsec-cli/metrics_table.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index f1e347e51b8..782af171fe6 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -247,7 +247,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error if _, ok := waap_rule_stats[waapEngine][ruleID]; !ok { waap_rule_stats[waapEngine][ruleID] = make(map[string]int, 0) } - waap_rule_stats[waapEngine][ruleID]["processed"] = ival + waap_rule_stats[waapEngine][ruleID]["triggered"] = ival default: log.Infof("unknown: %+v", fam.Name) continue diff --git a/cmd/crowdsec-cli/metrics_table.go b/cmd/crowdsec-cli/metrics_table.go index 51606cd9ce4..7df03318cfb 100644 --- a/cmd/crowdsec-cli/metrics_table.go +++ b/cmd/crowdsec-cli/metrics_table.go @@ -131,11 +131,11 @@ func waapRulesToTable(out io.Writer, metrics map[string]map[string]map[string]in for waapEngine, waapEngineRulesStats := range metrics { t := newTable(out) t.SetRowLines(false) - t.SetHeaders("Rule ID", "Processed") + t.SetHeaders("Rule ID", "Triggered") t.SetAlignment(table.AlignLeft, table.AlignLeft) - keys := []string{"processed"} + keys := []string{"triggered"} if numRows, err := metricsToTable(t, waapEngineRulesStats, keys); err != nil { - log.Warningf("while collecting waap stats: %s", err) + log.Warningf("while collecting waap rules stats: %s", err) } else if numRows > 0 { renderTableTitle(out, fmt.Sprintf("\nWaap '%s' Rules Metrics:", waapEngine)) t.Render() From 5ca2ee2f2ec29092f4ef821379ed26d6bcf7f635 Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 28 Nov 2023 15:10:32 +0100 Subject: [PATCH 208/263] update --- cmd/crowdsec-cli/metrics.go | 2 +- pkg/acquisition/modules/waap/metrics.go | 4 ++-- pkg/acquisition/modules/waap/utils.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 782af171fe6..c4fcb71c167 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -240,7 +240,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error waap_engine_stats[metric.Labels["waap_engine"]]["blocked"] = ival case "cs_waf_rule_hits": waapEngine := metric.Labels["waap_engine"] - ruleID := metric.Labels["rule_id"] + ruleID := metric.Labels["rule_name"] if _, ok := waap_rule_stats[waapEngine]; !ok { waap_rule_stats[waapEngine] = make(map[string]map[string]int, 0) } diff --git a/pkg/acquisition/modules/waap/metrics.go b/pkg/acquisition/modules/waap/metrics.go index 3545786f01f..fd917a8b2d3 100644 --- a/pkg/acquisition/modules/waap/metrics.go +++ b/pkg/acquisition/modules/waap/metrics.go @@ -48,7 +48,7 @@ var WafBlockCounter = prometheus.NewCounterVec( var WafRuleHits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cs_waf_rule_hits", - Help: "Count of triggered rule, by rule_id and type (inband/outofband).", + Help: "Count of triggered rule, by rule_name, type (inband/outofband), waap_engine and source", }, - []string{"rule_id", "type", "waap_engine", "source"}, + []string{"rule_name", "type", "waap_engine", "source"}, ) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index bd0e3b3a589..cf17e2638b8 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -201,22 +201,22 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest evt.Waap.HasOutBandMatches = true } - // TODO: Fetch the Name of the rule when possible - WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind, "source": req.RemoteAddrNormalized, "waap_engine": req.WaapEngine}).Inc() - name := "NOT_SET" version := "NOT_SET" hash := "NOT_SET" + ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID()) if details, ok := waf.WaapRulesDetails[rule.Rule().ID()]; ok { //Only set them for custom rules, not for rules written in seclang name = details.Name version = details.Version hash = details.Hash - + ruleNameProm = details.Name r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) } + WafRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "waap_engine": req.WaapEngine}).Inc() + corazaRule := map[string]interface{}{ "id": rule.Rule().ID(), "uri": evt.Parsed["uri"], From 8999154f766e2666df60d5826dd1d9747430792b Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 28 Nov 2023 11:02:29 +0100 Subject: [PATCH 209/263] up --- pkg/acquisition/modules/waap/waap.go | 4 +++- pkg/acquisition/modules/waap/waap_runner.go | 4 ++++ pkg/waf/waap.go | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index a478c2779bb..a35d0617593 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -364,7 +364,9 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { WafBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() } - waapResponse := w.WaapRuntime.GenerateResponse(response.InBandInterrupt) + w.logger.Infof("Response: %+v", response) + + waapResponse := w.WaapRuntime.GenerateResponse(response) rw.WriteHeader(waapResponse.HTTPStatus) body, err := json.Marshal(BodyResponse{Action: waapResponse.Action}) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 9a07f0966c7..93e0541607b 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -213,6 +213,7 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { continue } } + elapsed := time.Since(startParsing) WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) @@ -220,6 +221,9 @@ func (r *WaapRunner) Run(t *tomb.Tomb) error { //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation // send back the result to the HTTP handler for the InBand part + + r.logger.Infof("Response: %+v", r.WaapRuntime.Response) + request.ResponseChannel <- r.WaapRuntime.Response request.IsInBand = false diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index c72076db37b..54d10b67c3a 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -468,21 +468,21 @@ type BodyResponse struct { HTTPStatus int `json:"http_status"` } -func (w *WaapRuntimeConfig) GenerateResponse(interrupted bool) BodyResponse { +func (w *WaapRuntimeConfig) GenerateResponse(response WaapTempResponse) BodyResponse { resp := BodyResponse{} //if there is no interrupt, we should allow with default code - if !interrupted { + if !response.InBandInterrupt { resp.Action = w.Config.DefaultPassAction resp.HTTPStatus = w.Config.PassedHTTPCode return resp } - resp.Action = w.Response.Action + resp.Action = response.Action if resp.Action == "" { resp.Action = w.Config.DefaultRemediation } w.Logger.Debugf("action is %s", resp.Action) - resp.HTTPStatus = w.Response.HTTPResponseCode + resp.HTTPStatus = response.HTTPResponseCode if resp.HTTPStatus == 0 { resp.HTTPStatus = w.Config.BlockedHTTPCode } From b31d48a797d88252a57667727174c733447d96da Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 29 Nov 2023 16:23:49 +0100 Subject: [PATCH 210/263] rename headers --- pkg/waf/request.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 882a7142585..c2dc3e1ac75 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -7,17 +7,16 @@ import ( "net/http" "net/url" - "github.com/google/uuid" log "github.com/sirupsen/logrus" ) const ( - URIHeaderName = "X-Crowdsec-Waf-Uri" - VerbHeaderName = "X-Crowdsec-Waf-Verb" - HostHeaderName = "X-Crowdsec-Waf-Host" - IPHeaderName = "X-Crowdsec-Waf-Ip" - APIKeyHeaderName = "X-Crowdsec-Waf-Api-Key" + URIHeaderName = "X-Crowdsec-Waap-Uri" + VerbHeaderName = "X-Crowdsec-Waap-Verb" + HostHeaderName = "X-Crowdsec-Waap-Host" + IPHeaderName = "X-Crowdsec-Waap-Ip" + APIKeyHeaderName = "X-Crowdsec-Waap-Api-Key" ) // type ResponseRequest struct { From fe005f87e51c605aa9c4fac826badfe7a30de585 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 29 Nov 2023 16:52:24 +0100 Subject: [PATCH 211/263] up --- pkg/waf/waap.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 54d10b67c3a..89c14f52fe0 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -441,13 +441,11 @@ func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { //@tko how should we handle this ? it seems bouncer only understand bans, but it might be misleading ? case "deny", "ban", "block": w.Response.Action = "ban" - w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode case "log": w.Response.Action = action w.Response.HTTPResponseCode = w.Config.PassedHTTPCode case "captcha": w.Response.Action = action - w.Response.HTTPResponseCode = w.Config.BlockedHTTPCode default: return nil, fmt.Errorf("unknown action %s", action) } From 5f254769ae08fb606dac80b952115b164d0de4bf Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 29 Nov 2023 17:45:06 +0100 Subject: [PATCH 212/263] up --- pkg/acquisition/modules/waap/utils.go | 8 +- pkg/acquisition/modules/waap/waap_runner.go | 188 ++++++++++++-------- pkg/waf/waap.go | 3 +- 3 files changed, 118 insertions(+), 81 deletions(-) diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/waap/utils.go index cf17e2638b8..cf02abae719 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/waap/utils.go @@ -68,7 +68,7 @@ func WaapEventGeneration(inEvt types.Event) (*types.Event, error) { return &evt, nil } -func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { +func EventFromRequest(r *waf.ParsedRequest) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -81,7 +81,7 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { "target_uri": r.URI, "method": r.Method, "req_uuid": r.Tx.ID(), - "source": "coraza", + "source": "crowdsec-waap", //TBD: //http_status @@ -91,7 +91,7 @@ func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "coraza-waap"}, + Labels: map[string]string{"type": "crowdsec-waap"}, Process: true, Module: "waap", Src: "waap", @@ -130,7 +130,7 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { } -func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req waf.ParsedRequest) error { +func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequest) error { if evt == nil { //an error was already emitted, let's not spam the logs diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 93e0541607b..c2d48e6bc94 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -170,97 +170,133 @@ func (r *WaapRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { return err } -func (r *WaapRunner) Run(t *tomb.Tomb) error { - r.logger.Infof("Waap Runner ready to process event") - for { - select { - case <-t.Dying(): - r.logger.Infof("Waf Runner is dying") - return nil - case request := <-r.inChan: - r.logger.Debugf("Requests handled by runner %s", request.UUID) - r.WaapRuntime.ClearResponse() - - request.IsInBand = true - request.IsOutBand = false +func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { + //create the associated event for crowdsec itself + evt, err := EventFromRequest(request) + if err != nil { + //let's not interrupt the pipeline for this + r.logger.Errorf("unable to create event from request : %s", err) + } + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("inband rules matched : %d", in.RuleID) + r.WaapRuntime.Response.InBandInterrupt = true - //to measure the time spent in the WAF - startParsing := time.Now() + err = r.WaapRuntime.ProcessOnMatchRules(request) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + return + } + // Should the in band match trigger an event ? + if r.WaapRuntime.Response.SendEvent { + r.outChan <- evt + } - //inband WAAP rules - err := r.ProcessInBandRules(&request) - if err != nil { - r.logger.Errorf("unable to process InBand rules: %s", err) - continue - } - //create the associated event for crowdsec itself - evt, err := EventFromRequest(request) + // Should the in band match trigger an overflow ? + if r.WaapRuntime.Response.SendAlert { + waapOvlfw, err := WaapEventGeneration(evt) if err != nil { - //let's not interrupt the pipeline for this - r.logger.Errorf("unable to create event from request : %s", err) + r.logger.Errorf("unable to generate waap event : %s", err) + return } - err = r.AccumulateTxToEvent(&evt, request) + r.outChan <- *waapOvlfw + } + } +} + +func (r *WaapRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { + evt, err := EventFromRequest(request) + if err != nil { + //let's not interrupt the pipeline for this + r.logger.Errorf("unable to create event from request : %s", err) + } + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("inband rules matched : %d", in.RuleID) + r.WaapRuntime.Response.OutOfBandInterrupt = true + + err = r.WaapRuntime.ProcessOnMatchRules(request) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + return + } + // Should the match trigger an event ? + if r.WaapRuntime.Response.SendEvent { + r.outChan <- evt + } + + // Should the match trigger an overflow ? + if r.WaapRuntime.Response.SendAlert { + waapOvlfw, err := WaapEventGeneration(evt) if err != nil { - r.logger.Errorf("unable to accumulate tx to event : %s", err) - } - if in := request.Tx.Interruption(); in != nil { - r.logger.Debugf("inband rules matched : %d", in.RuleID) - r.WaapRuntime.Response.InBandInterrupt = true - - err = r.WaapRuntime.ProcessOnMatchRules(&request) - if err != nil { - r.logger.Errorf("unable to process OnMatch rules: %s", err) - continue - } + r.logger.Errorf("unable to generate waap event : %s", err) + return } + r.outChan <- *waapOvlfw + } + } +} - elapsed := time.Since(startParsing) - WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) +func (r *WaapRunner) handleRequest(request *waf.ParsedRequest) { + r.logger.Debugf("Requests handled by runner %s", request.UUID) + r.WaapRuntime.ClearResponse() - //generate reponse for the remediation component, based on the WAAP config + inband rules evaluation - //@tko : this should move in the WaapRuntimeConfig as it knows what to do with the interruption and the expected remediation + request.IsInBand = true + request.IsOutBand = false - // send back the result to the HTTP handler for the InBand part + //to measure the time spent in the WAF + startParsing := time.Now() - r.logger.Infof("Response: %+v", r.WaapRuntime.Response) + //inband WAAP rules + err := r.ProcessInBandRules(request) + if err != nil { + r.logger.Errorf("unable to process InBand rules: %s", err) + return + } - request.ResponseChannel <- r.WaapRuntime.Response + if request.Tx.IsInterrupted() { + r.handleInBandInterrupt(request) + } - request.IsInBand = false - request.IsOutBand = true + elapsed := time.Since(startParsing) + WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) - err = r.ProcessOutOfBandRules(&request) - if err != nil { - r.logger.Errorf("unable to process OutOfBand rules: %s", err) - continue - } - err = r.AccumulateTxToEvent(&evt, request) - if err != nil { - r.logger.Errorf("unable to accumulate tx to event : %s", err) - } - if in := request.Tx.Interruption(); in != nil { - r.logger.Debugf("outband rules matched : %d", in.RuleID) - r.WaapRuntime.Response.OutOfBandInterrupt = true - err = r.WaapRuntime.ProcessOnMatchRules(&request) - if err != nil { - r.logger.Errorf("unable to process OnMatch rules: %s", err) - continue - } - } + // send back the result to the HTTP handler for the InBand part + request.ResponseChannel <- r.WaapRuntime.Response - if !evt.Process { - continue - } + //Now let's process the out of band rules - //we generate two events: one that is going to be picked up by the acquisition pipeline (parsers, scenarios etc.) - //and a second one that will go straight to LAPI - r.outChan <- evt - waapOvlfw, err := WaapEventGeneration(evt) - if err != nil { - r.logger.Errorf("unable to generate waap event : %s", err) - } else if waapOvlfw != nil { - r.outChan <- *waapOvlfw - } + request.IsInBand = false + request.IsOutBand = true + r.WaapRuntime.Response.SendAlert = false + r.WaapRuntime.Response.SendEvent = true + + err = r.ProcessOutOfBandRules(request) + if err != nil { + r.logger.Errorf("unable to process OutOfBand rules: %s", err) + return + } + + if request.Tx.IsInterrupted() { + r.handleOutBandInterrupt(request) + } +} + +func (r *WaapRunner) Run(t *tomb.Tomb) error { + r.logger.Infof("Waap Runner ready to process event") + for { + select { + case <-t.Dying(): + r.logger.Infof("Waf Runner is dying") + return nil + case request := <-r.inChan: + r.handleRequest(&request) } } } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 89c14f52fe0..45a50c9c269 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -120,7 +120,8 @@ func (w *WaapRuntimeConfig) ClearResponse() { log.Debugf("-> %p", w.Config) w.Response.Action = w.Config.DefaultPassAction w.Response.HTTPResponseCode = w.Config.PassedHTTPCode - w.Response.SendEvent = true + w.Response.SendEvent = false + w.Response.SendAlert = true } func (wc *WaapConfig) LoadByPath(file string) error { From eed9ff0c4633d2b4fa00c558141aad5bd20b7a9a Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 29 Nov 2023 22:02:38 +0100 Subject: [PATCH 213/263] up --- pkg/waf/waap_rule/modsec_rule_test.go | 17 +++++++++-------- pkg/waf/waap_rule/modsecurity.go | 19 +++++++++++-------- pkg/waf/waf_helpers.go | 8 ++++---- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/pkg/waf/waap_rule/modsec_rule_test.go b/pkg/waf/waap_rule/modsec_rule_test.go index 62e04d8e154..2232540ec33 100644 --- a/pkg/waf/waap_rule/modsec_rule_test.go +++ b/pkg/waf/waap_rule/modsec_rule_test.go @@ -16,7 +16,7 @@ func TestVPatchRuleString(t *testing.T) { Match: match{Type: "regex", Value: "[^a-zA-Z]"}, Transform: []string{"lowercase"}, }, - expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1136235475,phase:2,deny,log,msg:'Base Rule',t:lowercase"`, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2203944045,phase:2,deny,log,msg:'Base Rule',tag:'crowdsec-Base Rule',t:lowercase"`, }, { name: "Multiple Zones", @@ -26,7 +26,7 @@ func TestVPatchRuleString(t *testing.T) { Match: match{Type: "regex", Value: "[^a-zA-Z]"}, Transform: []string{"lowercase"}, }, - expected: `SecRule ARGS_GET:foo|ARGS_POST:foo "@rx [^a-zA-Z]" "id:2088895799,phase:2,deny,log,msg:'Multiple Zones',t:lowercase"`, + expected: `SecRule ARGS_GET:foo|ARGS_POST:foo "@rx [^a-zA-Z]" "id:3387135861,phase:2,deny,log,msg:'Multiple Zones',tag:'crowdsec-Multiple Zones',t:lowercase"`, }, { name: "Basic AND", @@ -47,8 +47,8 @@ func TestVPatchRuleString(t *testing.T) { }, }, }, - expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2323451654,phase:2,deny,log,msg:'Basic AND_and_0',t:lowercase,chain" -SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2075918819,phase:2,deny,log,msg:'Basic AND_and_1',t:lowercase"`, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:4145519614,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase,chain" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1865217529,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase"`, }, { name: "Basic OR", @@ -68,8 +68,8 @@ SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2075918819,phase:2,deny,log,msg:'Basic }, }, }, - expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2720972114,phase:2,deny,log,msg:'Basic OR_or_0',t:lowercase,skip:1" -SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2638639999,phase:2,deny,log,msg:'Basic OR_or_1',t:lowercase"`, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:651140804,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:271441587,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase"`, }, { name: "OR AND mix", @@ -97,8 +97,9 @@ SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2638639999,phase:2,deny,log,msg:'Basic }, }, }, - expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2720972114,phase:2,deny,log,msg:'Basic OR_or_0',t:lowercase,skip:1" -SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:2638639999,phase:2,deny,log,msg:'Basic OR_or_1',t:lowercase"`, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1714963250,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase" +SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase"`, }, } diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go index 8b3ab9f9ffe..7060ebf1f37 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/waf/waap_rule/modsecurity.go @@ -52,7 +52,7 @@ var bodyTypeMatch map[string]string = map[string]string{ func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, []uint32, error) { - rules, err := m.buildRules(rule, waapRuleName, false, 0) + rules, err := m.buildRules(rule, waapRuleName, false, 0, 0) if err != nil { return "", nil, err @@ -62,11 +62,12 @@ func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, return strings.Join(rules, "\n"), m.ids, nil } -func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) uint32 { +func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string, depth int) uint32 { h := fnv.New32a() h.Write([]byte(waapRuleName)) h.Write([]byte(rule.Match.Type)) h.Write([]byte(rule.Match.Value)) + h.Write([]byte(fmt.Sprintf("%d", depth))) for _, zone := range rule.Zones { h.Write([]byte(zone)) } @@ -78,14 +79,15 @@ func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string) return id } -func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool, toSkip int) ([]string, error) { +func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool, toSkip int, depth int) ([]string, error) { ret := make([]string, 0) if rule.And != nil { for c, andRule := range rule.And { - subName := fmt.Sprintf("%s_and_%d", waapRuleName, c) + depth++ + //subName := fmt.Sprintf("%s_and_%d", waapRuleName, c) lastRule := c == len(rule.And)-1 // || len(rule.Or) == 0 - rules, err := m.buildRules(&andRule, subName, !lastRule, 0) + rules, err := m.buildRules(&andRule, waapRuleName, !lastRule, 0, depth) if err != nil { return nil, err } @@ -95,9 +97,10 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and if rule.Or != nil { for c, orRule := range rule.Or { - subName := fmt.Sprintf("%s_or_%d", waapRuleName, c) + depth++ + //subName := fmt.Sprintf("%s_or_%d", waapRuleName, c) skip := len(rule.Or) - c - 1 - rules, err := m.buildRules(&orRule, subName, false, skip) + rules, err := m.buildRules(&orRule, waapRuleName, false, skip, depth) if err != nil { return nil, err } @@ -140,7 +143,7 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and } //Should phase:2 be configurable? - r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s'`, m.generateRuleID(rule, waapRuleName), waapRuleName)) + r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s',tag:'crowdsec-%s'`, m.generateRuleID(rule, waapRuleName, depth), waapRuleName, waapRuleName)) if rule.Transform != nil { for _, transform := range rule.Transform { diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 968d8decadb..a97af1e816d 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -34,10 +34,10 @@ func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ - "DisableInBandRuleByID": w.DisableInBandRuleByID, - "DisableOutBandRuleByID": w.DisableOutBandRuleByID, - "DisableInBandRuleByTag": w.DisableInBandRuleByTag, - "DisableOutBandRuleByTag": w.DisableOutBandRuleByTag, + "RemoveInBandRuleByID": w.DisableInBandRuleByID, + "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, + "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, + "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, } } From 008480420cb655ff048577356e92695ff86f6d42 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 30 Nov 2023 16:28:52 +0100 Subject: [PATCH 214/263] typo --- pkg/waf/waf_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index a97af1e816d..c200d70b870 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -49,7 +49,7 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte "RemoveInBandRuleByID": w.RemoveInbandRuleByID, "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, "SetRemediationByTag": w.SetActionByTag, - "SetRemdiationByID": w.SetActionByID, + "SetRemediationByID": w.SetActionByID, } } From 0cd2a2da2097166a18886d1c9d06cf36cd367b95 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 30 Nov 2023 16:45:26 +0100 Subject: [PATCH 215/263] fix http code and remediation --- pkg/acquisition/modules/waap/waap_runner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index c2d48e6bc94..abbe32c6fcb 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -184,6 +184,8 @@ func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("inband rules matched : %d", in.RuleID) r.WaapRuntime.Response.InBandInterrupt = true + r.WaapRuntime.Response.HTTPResponseCode = r.WaapRuntime.Config.BlockedHTTPCode + r.WaapRuntime.Response.Action = r.WaapRuntime.DefaultRemediation err = r.WaapRuntime.ProcessOnMatchRules(request) if err != nil { From 1eab34eb3f853f63e4ad9d85f100ed47e276b107 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 1 Dec 2023 11:16:01 +0100 Subject: [PATCH 216/263] send event for in-band match --- pkg/waf/waap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 45a50c9c269..db0dce691e5 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -120,7 +120,7 @@ func (w *WaapRuntimeConfig) ClearResponse() { log.Debugf("-> %p", w.Config) w.Response.Action = w.Config.DefaultPassAction w.Response.HTTPResponseCode = w.Config.PassedHTTPCode - w.Response.SendEvent = false + w.Response.SendEvent = true w.Response.SendAlert = true } From a258cc0b4a25ae07c439aa66be791edd43f36eb4 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 1 Dec 2023 13:22:44 +0100 Subject: [PATCH 217/263] default waap path to / --- pkg/acquisition/modules/waap/waap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index a35d0617593..1c8eb8be807 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -113,7 +113,7 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { } if wc.config.Path == "" { - return fmt.Errorf("path cannot be empty") + wc.config.Path = "/" } if wc.config.Path[0] != '/' { From 68148e031cec20e3ff57a2f7c6be65d49c1f8dfd Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 1 Dec 2023 14:04:18 +0100 Subject: [PATCH 218/263] add evt to on_match hoks --- pkg/acquisition/modules/waap/waap_runner.go | 4 ++-- pkg/waf/waap.go | 9 +++++---- pkg/waf/waf_helpers.go | 4 +++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index abbe32c6fcb..935b3271680 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -187,7 +187,7 @@ func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { r.WaapRuntime.Response.HTTPResponseCode = r.WaapRuntime.Config.BlockedHTTPCode r.WaapRuntime.Response.Action = r.WaapRuntime.DefaultRemediation - err = r.WaapRuntime.ProcessOnMatchRules(request) + err = r.WaapRuntime.ProcessOnMatchRules(request, evt) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) return @@ -223,7 +223,7 @@ func (r *WaapRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { r.logger.Debugf("inband rules matched : %d", in.RuleID) r.WaapRuntime.Response.OutOfBandInterrupt = true - err = r.WaapRuntime.ProcessOnMatchRules(request) + err = r.WaapRuntime.ProcessOnMatchRules(request, evt) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) return diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index db0dce691e5..0954830a7ba 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -8,6 +8,7 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -37,7 +38,7 @@ func (h *Hook) Build(hookStage int) error { case hookPreEval: ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) case hookOnMatch: - ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) } opts := GetExprWAFOptions(ctx) if h.Filter != "" { @@ -285,11 +286,11 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { return nil } -func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest) error { +func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetOnMatchEnv(w, request)) + output, err := expr.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt)) if err != nil { return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) } @@ -305,7 +306,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest) error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetOnMatchEnv(w, request)) + _, err := expr.Run(applyExpr, GetOnMatchEnv(w, request, evt)) if err != nil { log.Errorf("unable to apply waap on_match expr: %s", err) continue diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index c200d70b870..dda66a6b2cf 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -3,6 +3,7 @@ package waf import ( "github.com/antonmedv/expr" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) var exprFunctionOptions []expr.Option @@ -53,9 +54,10 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte } } -func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { +func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ + "evt": evt, "req": request, "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, From 3836780d908f4abc17e41d348ca8fdce531e6b37 Mon Sep 17 00:00:00 2001 From: bui Date: Fri, 1 Dec 2023 14:12:57 +0100 Subject: [PATCH 219/263] up --- pkg/waf/waf_helpers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index c200d70b870..2973cfd7f0b 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -65,5 +65,6 @@ func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte "SendEvent": w.SendEvent, "CancelAlert": w.CancelAlert, "SendAlert": w.SendAlert, + "DumpRequest": request.DumpRequest, } } From 7e1fd33c7ebbc0f09dd52147171db846de313a87 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Fri, 1 Dec 2023 14:20:36 +0100 Subject: [PATCH 220/263] enable expr debugging for hooks --- pkg/waf/waap.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 0954830a7ba..7f7b198e28e 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -8,6 +8,7 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -260,7 +261,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetOnLoadEnv(w)) + output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_load filter %s : %w", rule.Filter, err) } @@ -276,7 +277,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetOnLoadEnv(w)) + _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_load expr: %s", err) continue @@ -290,7 +291,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt)) + output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) } @@ -306,7 +307,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type } } for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetOnMatchEnv(w, request, evt)) + _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_match expr: %s", err) continue @@ -319,7 +320,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := expr.Run(rule.FilterExpr, GetPreEvalEnv(w, request)) + output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap pre_eval filter %s : %w", rule.Filter, err) } @@ -336,7 +337,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := expr.Run(applyExpr, GetPreEvalEnv(w, request)) + _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap pre_eval expr: %s", err) continue From 17cfc9909e0a457bd2abb0de014d9be8b0ba1d67 Mon Sep 17 00:00:00 2001 From: bui Date: Mon, 4 Dec 2023 09:45:47 +0100 Subject: [PATCH 221/263] add request dumper with filters --- pkg/waf/request.go | 305 ++++++++++++++++++++++++++++++++-------- pkg/waf/request_test.go | 181 ++++++++++++++++++++++++ 2 files changed, 425 insertions(+), 61 deletions(-) create mode 100644 pkg/waf/request_test.go diff --git a/pkg/waf/request.go b/pkg/waf/request.go index c2dc3e1ac75..ecd8e647291 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -1,11 +1,14 @@ package waf import ( + "encoding/json" "fmt" "io" "net" "net/http" "net/url" + "os" + "regexp" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -19,68 +22,248 @@ const ( APIKeyHeaderName = "X-Crowdsec-Waap-Api-Key" ) -// type ResponseRequest struct { -// UUID string -// Tx corazatypes.Transaction -// Interruption *corazatypes.Interruption -// Err error -// SendEvents bool -// } - -// func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest { -// return ResponseRequest{ -// UUID: UUID, -// Tx: Tx, -// Interruption: in, -// Err: err, -// SendEvents: true, -// } -// } - -// func (r *ResponseRequest) SetRemediation(remediation string) error { -// if r.Interruption == nil { -// return nil -// } -// r.Interruption.Action = remediation -// return nil -// } - -// func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error { -// if r.Interruption == nil { -// return nil -// } -// if r.Interruption.RuleID == ID { -// r.Interruption.Action = remediation -// } -// return nil -// } - -// func (r *ResponseRequest) CancelEvent() error { -// // true by default -// r.SendEvents = false -// return nil -// } - type ParsedRequest struct { - RemoteAddr string - Host string - ClientIP string - URI string - Args url.Values - ClientHost string - Headers http.Header - URL *url.URL - Method string - Proto string - Body []byte - TransferEncoding []string - UUID string - Tx ExtendedTransaction - ResponseChannel chan WaapTempResponse - IsInBand bool - IsOutBand bool - WaapEngine string - RemoteAddrNormalized string + RemoteAddr string `json:"remote_addr,omitempty"` + Host string `json:"host,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + URI string `json:"uri,omitempty"` + Args url.Values `json:"args,omitempty"` + ClientHost string `json:"client_host,omitempty"` + Headers http.Header `json:"headers,omitempty"` + URL *url.URL `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Proto string `json:"proto,omitempty"` + Body []byte `json:"body,omitempty"` + TransferEncoding []string `json:"transfer_encoding,omitempty"` + UUID string `json:"uuid,omitempty"` + Tx ExtendedTransaction `json:"transaction,omitempty"` + ResponseChannel chan WaapTempResponse `json:"-"` + IsInBand bool `json:"-"` + IsOutBand bool `json:"-"` + WaapEngine string `json:"waap_engine,omitempty"` + RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` +} + +type ReqDumpFilter struct { + req *ParsedRequest + HeadersContentFilters []string + HeadersNameFilters []string + HeadersDrop bool + + BodyDrop bool + //BodyContentFilters []string TBD + + ArgsContentFilters []string + ArgsNameFilters []string + ArgsDrop bool +} + +func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { + filter := ReqDumpFilter{} + filter.BodyDrop = true + filter.HeadersNameFilters = []string{"cookie", "authorization"} + filter.req = r + return &filter +} + +// clear filters +func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter { + r2 := ReqDumpFilter{} + r2.req = r.req + return &r2 +} + +func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter { + r.HeadersContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithHeadersContentFilters(filter string) *ReqDumpFilter { + r.HeadersContentFilters = append(r.HeadersContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter { + r.HeadersNameFilters = append(r.HeadersNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter { + r.HeadersDrop = true + return r +} + +func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter { + r.HeadersDrop = false + r.HeadersNameFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithBody() *ReqDumpFilter { + r.BodyDrop = false + return r +} + +func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter { + r.BodyDrop = true + return r +} + +func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter { + r.ArgsContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithArgsContentFilters(filter string) *ReqDumpFilter { + r.ArgsContentFilters = append(r.ArgsContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter { + r.ArgsNameFilters = append(r.ArgsNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error { + if r.BodyDrop { + return nil + } + out.Body = r.req.Body + return nil +} + +func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error { + if r.ArgsDrop { + return nil + } + if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 { + out.Args = r.req.Args + return nil + } + out.Args = make(url.Values) + for k, vals := range r.req.Args { + reject := false + //exclude by match on name + for _, filter := range r.ArgsNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.ArgsContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Args[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error { + if r.HeadersDrop { + return nil + } + + if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 { + out.Headers = r.req.Headers + return nil + } + + out.Headers = make(http.Header) + for k, vals := range r.req.Headers { + reject := false + //exclude by match on name + for _, filter := range r.HeadersNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.HeadersContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Headers[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest { + //if there are no filters, we return the original request + if len(r.HeadersContentFilters) == 0 && + len(r.HeadersNameFilters) == 0 && + len(r.ArgsContentFilters) == 0 && + len(r.ArgsNameFilters) == 0 && + !r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop { + log.Warningf("no filters, returning original request") + return r.req + } + + r2 := ParsedRequest{} + r.FilterHeaders(&r2) + r.FilterBody(&r2) + r.FilterArgs(&r2) + return &r2 +} + +func (r *ReqDumpFilter) ToJSON() error { + fd, err := os.CreateTemp("/tmp/", "crowdsec_req_dump_*.json") + if err != nil { + return fmt.Errorf("while creating temp file: %w", err) + } + defer fd.Close() + enc := json.NewEncoder(fd) + enc.SetIndent("", " ") + + req := r.GetFilteredRequest() + + log.Warningf("dumping : %+v", req) + + if err := enc.Encode(req); err != nil { + return fmt.Errorf("while encoding request: %w", err) + } + log.Warningf("request dumped to %s", fd.Name()) + return nil } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine diff --git a/pkg/waf/request_test.go b/pkg/waf/request_test.go new file mode 100644 index 00000000000..2625e11f5d3 --- /dev/null +++ b/pkg/waf/request_test.go @@ -0,0 +1,181 @@ +package waf + +import "testing" + +func TestBodyDumper(t *testing.T) { + + tests := []struct { + name string + req *ParsedRequest + expect *ParsedRequest + filter func(r *ReqDumpFilter) *ReqDumpFilter + }{ + { + name: "default filter (cookie+authorization stripped + no body)", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r + }, + }, + { + name: "explicit empty filter", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.NoFilters() + }, + }, + { + name: "filter header", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoBody().WithHeadersNameFilter("test2") + }, + }, + { + name: "filter header content", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeadersContentFilters("tata") + }, + }, + { + name: "with headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeaders() + }, + }, + { + name: "drop headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoHeaders() + }, + }, + { + name: "with body", + req: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithBody() + }, + }, + { + name: "with empty args filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyArgsFilters() + }, + }, + { + name: "with args name filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsNameFilter("toto") + }, + }, + { + name: "WithEmptyHeadersFilters", + req: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyHeadersFilters() + }, + }, + { + name: "WithArgsContentFilters", + req: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}, "test2": {"toto"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsContentFilters("toto") + }, + }, + } + + for idx, test := range tests { + + t.Run(test.name, func(t *testing.T) { + orig_dr := test.req.DumpRequest() + result := test.filter(orig_dr).GetFilteredRequest() + + if len(result.Body) != len(test.expect.Body) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Body), len(test.expect.Body)) + } + if len(result.Headers) != len(test.expect.Headers) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Headers), len(test.expect.Headers)) + } + for k, v := range result.Headers { + if len(v) != len(test.expect.Headers[k]) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(v), len(test.expect.Headers[k])) + } + } + }) + } + +} From d9355e8c3acfd4898ef1f4076b649fe437ffec19 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 10:07:16 +0100 Subject: [PATCH 222/263] fix hubtest for waap --- pkg/hubtest/coverage.go | 6 +++--- pkg/hubtest/hubtest_item.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 570750b8e81..ad7ceff635f 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -20,12 +20,12 @@ type Coverage struct { } func (h *HubTest) GetWaapCoverage() ([]Coverage, error) { - if _, ok := h.HubIndex.Items[cwhub.WAAP_RULES]; !ok { + if len(h.HubIndex.GetItemMap(cwhub.WAAP_RULES)) == 0 { return nil, fmt.Errorf("no waap rules in hub index") } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.WAAP_RULES]) + pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.WAAP_RULES)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { @@ -165,7 +165,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { } func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { - if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { + if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { return nil, fmt.Errorf("no scenarios in hub index") } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index e59fa72e999..90cd3bd02a4 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -320,7 +320,7 @@ func (t *HubTestItem) InstallHub() error { continue } - if hubWaapRule, ok := t.HubIndex.Items[cwhub.WAAP_RULES][waaprule]; ok { + if hubWaapRule, ok := t.HubIndex.GetItemMap(cwhub.WAAP_RULES)[waaprule]; ok { waapRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubWaapRule.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", waapRuleSource, err) From 60faeaa7d75f104c2953423110cc547694291abf Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 10:29:14 +0100 Subject: [PATCH 223/263] add post_eval hook --- pkg/acquisition/modules/waap/waap_runner.go | 8 +++- pkg/waf/waap.go | 50 +++++++++++++++++++-- pkg/waf/waf_helpers.go | 9 ++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 935b3271680..3590f417da2 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -106,7 +106,7 @@ func (r *WaapRunner) processRequest(tx waf.ExtendedTransaction, request *waf.Par } } - request.Tx.ProcessURI(request.URI, request.Method, request.Proto) //TODO: The doc mentions that GET args needs to be added, but we never call AddArguments ? + request.Tx.ProcessURI(request.URI, request.Method, request.Proto) for k, vr := range request.Headers { for _, v := range vr { @@ -150,7 +150,11 @@ func (r *WaapRunner) processRequest(tx waf.ExtendedTransaction, request *waf.Par if in != nil { r.logger.Debugf("rules matched for body : %d", in.RuleID) - return nil + } + + err = r.WaapRuntime.ProcessPostEvalRules(request) + if err != nil { + r.logger.Errorf("unable to process PostEval rules: %s", err) } return nil diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 7f7b198e28e..f47f3312ab4 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -26,6 +26,7 @@ type Hook struct { const ( hookOnLoad = iota hookPreEval + hookPostEval hookOnMatch ) @@ -38,6 +39,8 @@ func (h *Hook) Build(hookStage int) error { ctx = GetOnLoadEnv(&WaapRuntimeConfig{}) case hookPreEval: ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + case hookPostEval: + ctx = GetPostEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) case hookOnMatch: ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) } @@ -83,6 +86,7 @@ type WaapRuntimeConfig struct { DefaultRemediation string CompiledOnLoad []Hook CompiledPreEval []Hook + CompiledPostEval []Hook CompiledOnMatch []Hook CompiledVariablesTracking []*regexp.Regexp Config *WaapConfig @@ -107,6 +111,7 @@ type WaapConfig struct { PassedHTTPCode int `yaml:"passed_http_code"` OnLoad []Hook `yaml:"on_load"` PreEval []Hook `yaml:"pre_eval"` + PostEval []Hook `yaml:"post_eval"` OnMatch []Hook `yaml:"on_match"` VariablesTracking []string `yaml:"variables_tracking"` InbandOptions WaapSubEngineOpts `yaml:"inband_options"` @@ -239,6 +244,14 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { ret.CompiledPreEval = append(ret.CompiledPreEval, hook) } + for _, hook := range wc.PostEval { + err := hook.Build(hookPostEval) + if err != nil { + return nil, fmt.Errorf("unable to build post_eval hook : %s", err) + } + ret.CompiledPostEval = append(ret.CompiledPostEval, hook) + } + for _, hook := range wc.OnMatch { err := hook.Build(hookOnMatch) if err != nil { @@ -268,7 +281,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { switch t := output.(type) { case bool: if !t { - log.Infof("filter didnt match") + log.Debugf("filter didnt match") continue } default: @@ -298,7 +311,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type switch t := output.(type) { case bool: if !t { - log.Infof("filter didnt match") + log.Debugf("filter didnt match") continue } default: @@ -327,7 +340,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { switch t := output.(type) { case bool: if !t { - log.Infof("filter didnt match") + log.Debugf("filter didnt match") continue } default: @@ -348,6 +361,37 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { return nil } +func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { + for _, rule := range w.CompiledPostEval { + if rule.FilterExpr != nil { + output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + return fmt.Errorf("unable to run waap post_eval filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Debugf("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range rule.ApplyExpr { + _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + log.Errorf("unable to apply waap post_eval expr: %s", err) + continue + } + } + } + + return nil +} + /* @sbl / @tko add the helpers to: - remove by id-range diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 83f33b795da..d485d59aa27 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -54,6 +54,15 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte } } +func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { + //FIXME: use expr.Function instead of this + return map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "DumpRequest": request.DumpRequest, + } +} + func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ From 2a920124feaaaee0f69275bbed146e191bcff5c0 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 11:08:58 +0100 Subject: [PATCH 224/263] return an error if a custom rule has both and and or --- pkg/waf/waap_rule/modsecurity.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/waf/waap_rule/modsecurity.go index 7060ebf1f37..1ac7486dbe2 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/waf/waap_rule/modsecurity.go @@ -82,6 +82,10 @@ func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string, func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool, toSkip int, depth int) ([]string, error) { ret := make([]string, 0) + if len(rule.And) != 0 && len(rule.Or) != 0 { + return nil, fmt.Errorf("cannot have both 'and' and 'or' in the same rule") + } + if rule.And != nil { for c, andRule := range rule.And { depth++ From 393a8b8ef5ff9ed05b2a7ed1f872da0bb47f1e9d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 11:31:31 +0100 Subject: [PATCH 225/263] linting --- pkg/acquisition/modules/waap/waap.go | 2 +- pkg/hubtest/hubtest_item.go | 7 +++-- pkg/types/waap_event.go | 2 +- pkg/waf/coraza_logger.go | 5 +--- pkg/waf/request.go | 19 +++++++------ pkg/waf/waf_expr_lib.go | 42 +--------------------------- pkg/waf/waf_helpers.go | 13 --------- 7 files changed, 19 insertions(+), 71 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 1c8eb8be807..8830ef993cb 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -305,7 +305,7 @@ func (w *WaapSource) IsAuth(apiKey string) bool { Timeout: 200 * time.Millisecond, } - req, err := http.NewRequest("HEAD", w.lapiURL, nil) + req, err := http.NewRequest(http.MethodHead, w.lapiURL, nil) if err != nil { log.Errorf("Error creating request: %s", err) return false diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 90cd3bd02a4..8e0bc1a201f 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -581,14 +581,17 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { //wait for the waap port to be available if _, err := IsAlive(DefaultWaapHost); err != nil { - return fmt.Errorf("Waap is down: %s", err) + return fmt.Errorf("waap is down: %s", err) } // check if the target is available nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget) + if err != nil { + return fmt.Errorf("unable to parse target '%s': %s", DefaultNucleiTarget, err) + } nucleiTargetHost := nucleiTargetParsedURL.Host if _, err := IsAlive(nucleiTargetHost); err != nil { - return fmt.Errorf("Target is down: %s", err) + return fmt.Errorf("target is down: %s", err) } nucleiConfig := NucleiConfig{ diff --git a/pkg/types/waap_event.go b/pkg/types/waap_event.go index c0c89408e6f..9b64d8d7a34 100644 --- a/pkg/types/waap_event.go +++ b/pkg/types/waap_event.go @@ -8,7 +8,7 @@ import ( /* 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban - 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban + 2. If user triggered 3 distinct rules with medium confidence across 3 different requests, ban any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) diff --git a/pkg/waf/coraza_logger.go b/pkg/waf/coraza_logger.go index 82e0a169774..d7e680c76d7 100644 --- a/pkg/waf/coraza_logger.go +++ b/pkg/waf/coraza_logger.go @@ -112,10 +112,7 @@ func (e *crzLogEvent) Stringer(key string, val fmt.Stringer) dbg.Event { } func (e crzLogEvent) IsEnabled() bool { - if e.muted { - return false - } - return true + return !e.muted } type crzLogger struct { diff --git a/pkg/waf/request.go b/pkg/waf/request.go index ecd8e647291..5e792a13bbf 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -281,22 +281,22 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { // the real source of the request is set in 'x-client-ip' clientIP := r.Header.Get(IPHeaderName) if clientIP == "" { - return ParsedRequest{}, fmt.Errorf("Missing '%s' header", IPHeaderName) + return ParsedRequest{}, fmt.Errorf("missing '%s' header", IPHeaderName) } // the real target Host of the request is set in 'x-client-host' clientHost := r.Header.Get(HostHeaderName) if clientHost == "" { - return ParsedRequest{}, fmt.Errorf("Missing '%s' header", HostHeaderName) + return ParsedRequest{}, fmt.Errorf("missing '%s' header", HostHeaderName) } // the real URI of the request is set in 'x-client-uri' clientURI := r.Header.Get(URIHeaderName) if clientURI == "" { - return ParsedRequest{}, fmt.Errorf("Missing '%s' header", URIHeaderName) + return ParsedRequest{}, fmt.Errorf("missing '%s' header", URIHeaderName) } // the real VERB of the request is set in 'x-client-uri' clientMethod := r.Header.Get(VerbHeaderName) if clientMethod == "" { - return ParsedRequest{}, fmt.Errorf("Missing '%s' header", VerbHeaderName) + return ParsedRequest{}, fmt.Errorf("missing '%s' header", VerbHeaderName) } // delete those headers before coraza process the request @@ -310,18 +310,19 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err) } - RemoteAddrNormalized := "" + remoteAddrNormalized := "" host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { log.Errorf("Invalid waap remote IP source %v: %s", r.RemoteAddr, err.Error()) - RemoteAddrNormalized = r.RemoteAddr + remoteAddrNormalized = r.RemoteAddr } else { ip := net.ParseIP(host) if ip == nil { log.Errorf("Invalid waap remote IP address source %v: %s", r.RemoteAddr, err.Error()) - RemoteAddrNormalized = r.RemoteAddr + remoteAddrNormalized = r.RemoteAddr + } else { + remoteAddrNormalized = ip.String() } - RemoteAddrNormalized = ip.String() } return ParsedRequest{ @@ -339,6 +340,6 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args TransferEncoding: r.TransferEncoding, ResponseChannel: make(chan WaapTempResponse), - RemoteAddrNormalized: RemoteAddrNormalized, + RemoteAddrNormalized: remoteAddrNormalized, }, nil } diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 23e2c711bec..717b1bdff46 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -8,44 +8,4 @@ type exprCustomFunc struct { signature []interface{} } -/* -func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { - return map[string]interface{}{ - "DisableInBandRuleByID": w.DisableInBandRuleByID, - "DisableOutBandRuleByID": w.DisableOutBandRuleByID, - "DisableInBandRuleByTag": w.DisableInBandRuleByTag, - "DisableOutBandRuleByTag": w.DisableOutBandRuleByTag, - } -} -*/ - -/*var onLoadExprFuncs = []exprCustomFunc{ - { - name: "DisableInBandRuleByID", - function: w.DisableInBandRuleByID, - signature: []interface{}{ - new(func(int) error), - }, - }, -}*/ - -var preEvalExprFuncs = []exprCustomFunc{} - -var onMatchExprFuncs = []exprCustomFunc{} - -var exprFuncs = []exprCustomFunc{ - /*{ - name: "SetRulesToInband", - function: SetRulesToInband, - signature: []interface{}{ - new(func() error), - }, - }, - { - name: "SetRulesToOutOfBand", - function: SetRulesToOutOfBand, - signature: []interface{}{ - new(func() error), - }, - },*/ -} +var exprFuncs = []exprCustomFunc{} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index d485d59aa27..137f8544b2b 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -6,19 +6,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -var exprFunctionOptions []expr.Option - -func initWafHelpers() { - exprFunctionOptions = []expr.Option{} - for _, function := range exprFuncs { - exprFunctionOptions = append(exprFunctionOptions, - expr.Function(function.name, - function.function, - function.signature..., - )) - } -} - func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { baseHelpers := exprhelpers.GetExprOptions(ctx) From 3d3bf0bb0ed646cf8b8e47965c15f089aa91501d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 11:46:01 +0100 Subject: [PATCH 226/263] lint --- pkg/exprhelpers/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index a8679beec4e..79a621c7d35 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -203,7 +203,7 @@ func Distinct(params ...any) (any, error) { } func FlattenDistinct(params ...any) (any, error) { - return Distinct(flatten(nil, reflect.ValueOf(params))) + return Distinct(flatten(nil, reflect.ValueOf(params))) //nolint:asasalint } func Flatten(params ...any) (any, error) { From 6fb965bb3f3d64677abe2942f7d6bf7f0444ac5e Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 14:01:10 +0100 Subject: [PATCH 227/263] add SetRemediationByTag/Name/ID --- go.mod | 10 +-- go.sum | 10 +++ pkg/acquisition/modules/waap/waap_runner.go | 11 ++++ pkg/waf/waap.go | 69 ++++++++++++++++++--- pkg/waf/waf_helpers.go | 4 ++ 5 files changed, 91 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 7db22663f16..2d208484a2d 100644 --- a/go.mod +++ b/go.mod @@ -79,9 +79,9 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 github.com/xhit/go-simple-mail/v2 v2.16.0 - golang.org/x/crypto v0.15.0 + golang.org/x/crypto v0.16.0 golang.org/x/mod v0.11.0 - golang.org/x/sys v0.14.0 + golang.org/x/sys v0.15.0 google.golang.org/grpc v1.56.3 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -90,7 +90,7 @@ require ( ) require ( - github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 + github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.0 @@ -200,9 +200,9 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.18.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/sync v0.5.0 // indirect - golang.org/x/term v0.14.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index b3c93a70be0..085a77d13c0 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 h1:Kp1sY2PE1H5nbr7xgAQeEWDqDW/o3HNL1rHvcVqzWT4= github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 h1:THMSMkBW/DLG5NvMAr/Mdg/eQOrEnMJ9Y+UdFG4yV8k= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= @@ -754,6 +756,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -789,6 +793,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -841,6 +847,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -850,6 +858,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 3590f417da2..2a9e53c2964 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -3,6 +3,7 @@ package wafacquisition import ( "fmt" "os" + "slices" "time" "github.com/crowdsecurity/coraza/v3" @@ -191,6 +192,16 @@ func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { r.WaapRuntime.Response.HTTPResponseCode = r.WaapRuntime.Config.BlockedHTTPCode r.WaapRuntime.Response.Action = r.WaapRuntime.DefaultRemediation + if _, ok := r.WaapRuntime.RemediationById[in.RuleID]; ok { + r.WaapRuntime.Response.Action = r.WaapRuntime.RemediationById[in.RuleID] + } + + for tag, remediation := range r.WaapRuntime.RemediationByTag { + if slices.Contains[[]string, string](in.Tags, tag) { + r.WaapRuntime.Response.Action = remediation + } + } + err = r.WaapRuntime.ProcessOnMatchRules(request, evt) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index f47f3312ab4..28abf6b047b 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -84,6 +84,8 @@ type WaapRuntimeConfig struct { InBandRules []WaapCollection DefaultRemediation string + RemediationByTag map[string]string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + RemediationById map[int]string CompiledOnLoad []Hook CompiledPreEval []Hook CompiledPostEval []Hook @@ -99,6 +101,13 @@ type WaapRuntimeConfig struct { //should we store matched rules here ? Logger *log.Entry + + //Set by on_load to ignore some rules on loading + disabledInBandRuleIds []int + disabledInBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + + disabledOutOfBandRuleIds []int + disabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME } type WaapConfig struct { @@ -423,26 +432,46 @@ func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { } // func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { +// Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableInBandRuleByID(params ...any) (any, error) { - panic("not implemented") + w.disabledInBandRuleIds = append(w.disabledInBandRuleIds, params[0].(int)) + return nil, nil +} + +// func (w *WaapRuntimeConfig) DisableInBandRuleByName(name string) error { +// Disable a rule at load time, meaning it will not run for any request +func (w *WaapRuntimeConfig) DisableInBandRuleByName(params ...any) (any, error) { + tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) + w.disabledInBandRulesTags = append(w.disabledInBandRulesTags, tagValue) return nil, nil } -// func (w *WaapRuntimeConfig) DisableInBandRuleByTag(id int) error { +// func (w *WaapRuntimeConfig) DisableInBandRuleByTag(tag string) error { +// Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableInBandRuleByTag(params ...any) (any, error) { - panic("not implemented") + w.disabledInBandRulesTags = append(w.disabledInBandRulesTags, params[0].(string)) return nil, nil } -// func (w *WaapRuntimeConfig) DisableOutBandRuleByID(tag string) error { +// func (w *WaapRuntimeConfig) DisableOutBandRuleByID(id int) error { +// Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableOutBandRuleByID(params ...any) (any, error) { - panic("not implemented") + w.disabledOutOfBandRuleIds = append(w.disabledOutOfBandRuleIds, params[0].(int)) + return nil, nil +} + +// func (w *WaapRuntimeConfig) DisableOutBandRuleByName(name string) error { +// Disable a rule at load time, meaning it will not run for any request +func (w *WaapRuntimeConfig) DisableOutBandRuleByName(params ...any) (any, error) { + tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) + w.disabledOutOfBandRulesTags = append(w.disabledOutOfBandRulesTags, tagValue) return nil, nil } // func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { +// Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(params ...any) (any, error) { - panic("not implemented") + w.disabledOutOfBandRulesTags = append(w.disabledOutOfBandRulesTags, params[0].(string)) return nil, nil } @@ -466,13 +495,37 @@ func (w *WaapRuntimeConfig) CancelAlert(params ...any) (any, error) { // func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { func (w *WaapRuntimeConfig) SetActionByTag(params ...any) (any, error) { - panic("not implemented") + if w.RemediationByTag == nil { + w.RemediationByTag = make(map[string]string) + } + tag := params[0].(string) + action := params[1].(string) + w.Logger.Debugf("setting action of %s to %s", tag, action) + w.RemediationByTag[tag] = action return nil, nil } // func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { - panic("not implemented") + if w.RemediationById == nil { + w.RemediationById = make(map[int]string) + } + id := params[0].(int) + action := params[1].(string) + w.Logger.Debugf("setting action of %d to %s", id, action) + w.RemediationById[id] = action + return nil, nil +} + +// func (w *WaapRuntimeConfig) SetActionByID(name string, action string) error { +func (w *WaapRuntimeConfig) SetActionByName(params ...any) (any, error) { + if w.RemediationByTag == nil { + w.RemediationByTag = make(map[string]string) + } + tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) + action := params[1].(string) + w.Logger.Debugf("setting action of %s to %s", tag, action) + w.RemediationByTag[tag] = action return nil, nil } diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 137f8544b2b..69b53df1ef1 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -26,6 +26,9 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, } } @@ -38,6 +41,7 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, "SetRemediationByTag": w.SetActionByTag, "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, } } From cb030beaca01cd7a30773c13bf61e70afdefe127 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 15:02:32 +0100 Subject: [PATCH 228/263] Fix Remove{in,out}bandby{name,tag} --- go.mod | 2 +- go.sum | 4 ++++ pkg/acquisition/modules/waap/waap_runner.go | 24 +++++++++++++++++++++ pkg/waf/waap.go | 20 ++++++++--------- pkg/waf/waf_helpers.go | 16 ++++++++------ 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2d208484a2d..68e7b6a4d3e 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( ) require ( - github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 + github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.0 diff --git a/go.sum b/go.sum index 085a77d13c0..10f86416098 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,10 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 h1:Kp1sY2P github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 h1:THMSMkBW/DLG5NvMAr/Mdg/eQOrEnMJ9Y+UdFG4yV8k= github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9 h1:vFJiYtKOW5DwGQ9gxQi8+XDNc+YvuXXsJyWXXuiOn+M= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 h1:vY0KZvoS4Xl9IfGucBA4l1CV1auRPPJtjZSTz/Rl6iQ= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/waap/waap_runner.go index 2a9e53c2964..5375a4d2c71 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/waap/waap_runner.go @@ -70,6 +70,30 @@ func (r *WaapRunner) Init(datadir string) error { } r.WaapOutbandEngine, err = coraza.NewWAF(outbandCfg) + if r.WaapRuntime.DisabledInBandRulesTags != nil { + for _, tag := range r.WaapRuntime.DisabledInBandRulesTags { + r.WaapInbandEngine.GetRuleGroup().DeleteByTag(tag) + } + } + + if r.WaapRuntime.DisabledOutOfBandRulesTags != nil { + for _, tag := range r.WaapRuntime.DisabledOutOfBandRulesTags { + r.WaapOutbandEngine.GetRuleGroup().DeleteByTag(tag) + } + } + + if r.WaapRuntime.DisabledInBandRuleIds != nil { + for _, id := range r.WaapRuntime.DisabledInBandRuleIds { + r.WaapInbandEngine.GetRuleGroup().DeleteByID(id) + } + } + + if r.WaapRuntime.DisabledOutOfBandRuleIds != nil { + for _, id := range r.WaapRuntime.DisabledOutOfBandRuleIds { + r.WaapOutbandEngine.GetRuleGroup().DeleteByID(id) + } + } + if err != nil { return fmt.Errorf("unable to initialize outband engine : %w", err) } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 28abf6b047b..85eb4e4fd50 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -103,11 +103,11 @@ type WaapRuntimeConfig struct { Logger *log.Entry //Set by on_load to ignore some rules on loading - disabledInBandRuleIds []int - disabledInBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + DisabledInBandRuleIds []int + DisabledInBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME - disabledOutOfBandRuleIds []int - disabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + DisabledOutOfBandRuleIds []int + DisabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME } type WaapConfig struct { @@ -434,7 +434,7 @@ func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { // func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableInBandRuleByID(params ...any) (any, error) { - w.disabledInBandRuleIds = append(w.disabledInBandRuleIds, params[0].(int)) + w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, params[0].(int)) return nil, nil } @@ -442,21 +442,21 @@ func (w *WaapRuntimeConfig) DisableInBandRuleByID(params ...any) (any, error) { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableInBandRuleByName(params ...any) (any, error) { tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) - w.disabledInBandRulesTags = append(w.disabledInBandRulesTags, tagValue) + w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tagValue) return nil, nil } // func (w *WaapRuntimeConfig) DisableInBandRuleByTag(tag string) error { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableInBandRuleByTag(params ...any) (any, error) { - w.disabledInBandRulesTags = append(w.disabledInBandRulesTags, params[0].(string)) + w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, params[0].(string)) return nil, nil } // func (w *WaapRuntimeConfig) DisableOutBandRuleByID(id int) error { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableOutBandRuleByID(params ...any) (any, error) { - w.disabledOutOfBandRuleIds = append(w.disabledOutOfBandRuleIds, params[0].(int)) + w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, params[0].(int)) return nil, nil } @@ -464,14 +464,14 @@ func (w *WaapRuntimeConfig) DisableOutBandRuleByID(params ...any) (any, error) { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableOutBandRuleByName(params ...any) (any, error) { tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) - w.disabledOutOfBandRulesTags = append(w.disabledOutOfBandRulesTags, tagValue) + w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tagValue) return nil, nil } // func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { // Disable a rule at load time, meaning it will not run for any request func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(params ...any) (any, error) { - w.disabledOutOfBandRulesTags = append(w.disabledOutOfBandRulesTags, params[0].(string)) + w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, params[0].(string)) return nil, nil } diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index 69b53df1ef1..bb65df851c8 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -22,13 +22,15 @@ func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ - "RemoveInBandRuleByID": w.DisableInBandRuleByID, - "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, - "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, - "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, - "SetRemediationByTag": w.SetActionByTag, - "SetRemediationByID": w.SetActionByID, - "SetRemediationByName": w.SetActionByName, + "RemoveInBandRuleByID": w.DisableInBandRuleByID, + "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, + "RemoveInBandRuleByName": w.DisableInBandRuleByName, + "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, + "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, + "RemoveOutBandRuleByName": w.DisableOutBandRuleByName, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, } } From b01901b04ea497608c7d6a2ca7ecd2c14c4633ea Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 15:13:11 +0100 Subject: [PATCH 229/263] fix Remove{in,out}bandRuleBy{name,tag} for pre_eval --- pkg/waf/tx.go | 5 +++++ pkg/waf/waap.go | 32 ++++++++++++++++++++++++++++++++ pkg/waf/waf_helpers.go | 22 +++++++++++++--------- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/pkg/waf/tx.go b/pkg/waf/tx.go index ce44aec95ec..1e2c83331d9 100644 --- a/pkg/waf/tx.go +++ b/pkg/waf/tx.go @@ -27,6 +27,11 @@ func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { return nil } +func (t *ExtendedTransaction) RemoveRuleByTagWithError(tag string) error { + t.Tx.RemoveRuleByTag(tag) + return nil +} + func (t *ExtendedTransaction) IsRuleEngineOff() bool { return t.Tx.IsRuleEngineOff() } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 85eb4e4fd50..2820bb8c496 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -425,6 +425,38 @@ func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { return nil, nil } +// func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(tag string) error { +func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(params ...any) (any, error) { + tag := params[0].(string) + w.Logger.Debugf("removing inband rule with tag %s", tag) + _ = w.InBandTx.RemoveRuleByTagWithError(tag) + return nil, nil +} + +// func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { +func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(params ...any) (any, error) { + tag := params[0].(string) + w.Logger.Debugf("removing outband rule with tag %s", tag) + _ = w.OutOfBandTx.RemoveRuleByTagWithError(tag) + return nil, nil +} + +// func (w *WaapRuntimeConfig) RemoveInbandRuleByName(name string) error { +func (w *WaapRuntimeConfig) RemoveInbandRuleByName(params ...any) (any, error) { + tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) + w.Logger.Debugf("removing inband rule %s", tag) + _ = w.InBandTx.RemoveRuleByTagWithError(tag) + return nil, nil +} + +// func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(name string) error { +func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(params ...any) (any, error) { + tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) + w.Logger.Debugf("removing outband rule %s", tag) + _ = w.OutOfBandTx.RemoveRuleByTagWithError(tag) + return nil, nil +} + func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { w.Logger.Debugf("canceling event") w.Response.SendEvent = false diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index bb65df851c8..ced1b68e787 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -23,9 +23,9 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ "RemoveInBandRuleByID": w.DisableInBandRuleByID, - "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, - "RemoveInBandRuleByName": w.DisableInBandRuleByName, "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, + "RemoveInBandRuleByName": w.DisableInBandRuleByName, + "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, "RemoveOutBandRuleByName": w.DisableOutBandRuleByName, "SetRemediationByTag": w.SetActionByTag, @@ -37,13 +37,17 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { //FIXME: use expr.Function instead of this return map[string]interface{}{ - "IsInBand": request.IsInBand, - "IsOutBand": request.IsOutBand, - "RemoveInBandRuleByID": w.RemoveInbandRuleByID, - "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, - "SetRemediationByTag": w.SetActionByTag, - "SetRemediationByID": w.SetActionByID, - "SetRemediationByName": w.SetActionByName, + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "RemoveInBandRuleByID": w.RemoveInbandRuleByID, + "RemoveInBandRuleByName": w.RemoveInbandRuleByName, + "RemoveInBandRuleByTag": w.RemoveInbandRuleByTag, + "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, + "RemoveOutBandRuleByTag": w.RemoveOutbandRuleByTag, + "RemoveOutBandRuleByName": w.RemoveOutbandRuleByName, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, } } From ac451ccaf355b6977131f1945a946d875e268705 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:00:09 +0100 Subject: [PATCH 230/263] use expr func --- pkg/exprhelpers/expr_lib.go | 382 ++++++++++++++++++------------------ pkg/exprhelpers/helpers.go | 6 +- pkg/waf/request.go | 4 +- pkg/waf/request_test.go | 3 +- pkg/waf/waap.go | 50 +++-- pkg/waf/waf_expr_lib.go | 8 - pkg/waf/waf_helpers.go | 299 +++++++++++++++++++++++++--- 7 files changed, 501 insertions(+), 251 deletions(-) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index db191b84a8d..b4f18a8a3ab 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -6,438 +6,438 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cticlient" ) -type exprCustomFunc struct { - name string - function func(params ...any) (any, error) - signature []interface{} +type ExprCustomFunc struct { + Name string + Function func(params ...any) (any, error) + Signature []interface{} } -var exprFuncs = []exprCustomFunc{ +var exprFuncs = []ExprCustomFunc{ { - name: "CrowdsecCTI", - function: CrowdsecCTI, - signature: []interface{}{ + Name: "CrowdsecCTI", + Function: CrowdsecCTI, + Signature: []interface{}{ new(func(string) (*cticlient.SmokeItem, error)), }, }, { - name: "Flatten", - function: Flatten, - signature: []interface{}{}, + Name: "Flatten", + Function: Flatten, + Signature: []interface{}{}, }, { - name: "Distinct", - function: Distinct, - signature: []interface{}{}, + Name: "Distinct", + Function: Distinct, + Signature: []interface{}{}, }, { - name: "FlattenDistinct", - function: FlattenDistinct, - signature: []interface{}{}, + Name: "FlattenDistinct", + Function: FlattenDistinct, + Signature: []interface{}{}, }, { - name: "Distance", - function: Distance, - signature: []interface{}{ + Name: "Distance", + Function: Distance, + Signature: []interface{}{ new(func(string, string, string, string) (float64, error)), }, }, { - name: "GetFromStash", - function: GetFromStash, - signature: []interface{}{ + Name: "GetFromStash", + Function: GetFromStash, + Signature: []interface{}{ new(func(string, string) (string, error)), }, }, { - name: "Atof", - function: Atof, - signature: []interface{}{ + Name: "Atof", + Function: Atof, + Signature: []interface{}{ new(func(string) float64), }, }, { - name: "JsonExtract", - function: JsonExtract, - signature: []interface{}{ + Name: "JsonExtract", + Function: JsonExtract, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "JsonExtractUnescape", - function: JsonExtractUnescape, - signature: []interface{}{ + Name: "JsonExtractUnescape", + Function: JsonExtractUnescape, + Signature: []interface{}{ new(func(string, ...string) string), }, }, { - name: "JsonExtractLib", - function: JsonExtractLib, - signature: []interface{}{ + Name: "JsonExtractLib", + Function: JsonExtractLib, + Signature: []interface{}{ new(func(string, ...string) string), }, }, { - name: "JsonExtractSlice", - function: JsonExtractSlice, - signature: []interface{}{ + Name: "JsonExtractSlice", + Function: JsonExtractSlice, + Signature: []interface{}{ new(func(string, string) []interface{}), }, }, { - name: "JsonExtractObject", - function: JsonExtractObject, - signature: []interface{}{ + Name: "JsonExtractObject", + Function: JsonExtractObject, + Signature: []interface{}{ new(func(string, string) map[string]interface{}), }, }, { - name: "ToJsonString", - function: ToJson, - signature: []interface{}{ + Name: "ToJsonString", + Function: ToJson, + Signature: []interface{}{ new(func(interface{}) string), }, }, { - name: "File", - function: File, - signature: []interface{}{ + Name: "File", + Function: File, + Signature: []interface{}{ new(func(string) []string), }, }, { - name: "RegexpInFile", - function: RegexpInFile, - signature: []interface{}{ + Name: "RegexpInFile", + Function: RegexpInFile, + Signature: []interface{}{ new(func(string, string) bool), }, }, { - name: "Upper", - function: Upper, - signature: []interface{}{ + Name: "Upper", + Function: Upper, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "Lower", - function: Lower, - signature: []interface{}{ + Name: "Lower", + Function: Lower, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "IpInRange", - function: IpInRange, - signature: []interface{}{ + Name: "IpInRange", + Function: IpInRange, + Signature: []interface{}{ new(func(string, string) bool), }, }, { - name: "TimeNow", - function: TimeNow, - signature: []interface{}{ + Name: "TimeNow", + Function: TimeNow, + Signature: []interface{}{ new(func() string), }, }, { - name: "ParseUri", - function: ParseUri, - signature: []interface{}{ + Name: "ParseUri", + Function: ParseUri, + Signature: []interface{}{ new(func(string) map[string][]string), }, }, { - name: "PathUnescape", - function: PathUnescape, - signature: []interface{}{ + Name: "PathUnescape", + Function: PathUnescape, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "QueryUnescape", - function: QueryUnescape, - signature: []interface{}{ + Name: "QueryUnescape", + Function: QueryUnescape, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "PathEscape", - function: PathEscape, - signature: []interface{}{ + Name: "PathEscape", + Function: PathEscape, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "QueryEscape", - function: QueryEscape, - signature: []interface{}{ + Name: "QueryEscape", + Function: QueryEscape, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "XMLGetAttributeValue", - function: XMLGetAttributeValue, - signature: []interface{}{ + Name: "XMLGetAttributeValue", + Function: XMLGetAttributeValue, + Signature: []interface{}{ new(func(string, string, string) string), }, }, { - name: "XMLGetNodeValue", - function: XMLGetNodeValue, - signature: []interface{}{ + Name: "XMLGetNodeValue", + Function: XMLGetNodeValue, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "IpToRange", - function: IpToRange, - signature: []interface{}{ + Name: "IpToRange", + Function: IpToRange, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "IsIPV6", - function: IsIPV6, - signature: []interface{}{ + Name: "IsIPV6", + Function: IsIPV6, + Signature: []interface{}{ new(func(string) bool), }, }, { - name: "IsIPV4", - function: IsIPV4, - signature: []interface{}{ + Name: "IsIPV4", + Function: IsIPV4, + Signature: []interface{}{ new(func(string) bool), }, }, { - name: "IsIP", - function: IsIP, - signature: []interface{}{ + Name: "IsIP", + Function: IsIP, + Signature: []interface{}{ new(func(string) bool), }, }, { - name: "LookupHost", - function: LookupHost, - signature: []interface{}{ + Name: "LookupHost", + Function: LookupHost, + Signature: []interface{}{ new(func(string) []string), }, }, { - name: "GetDecisionsCount", - function: GetDecisionsCount, - signature: []interface{}{ + Name: "GetDecisionsCount", + Function: GetDecisionsCount, + Signature: []interface{}{ new(func(string) int), }, }, { - name: "GetDecisionsSinceCount", - function: GetDecisionsSinceCount, - signature: []interface{}{ + Name: "GetDecisionsSinceCount", + Function: GetDecisionsSinceCount, + Signature: []interface{}{ new(func(string, string) int), }, }, { - name: "Sprintf", - function: Sprintf, - signature: []interface{}{ + Name: "Sprintf", + Function: Sprintf, + Signature: []interface{}{ new(func(string, ...interface{}) string), }, }, { - name: "ParseUnix", - function: ParseUnix, - signature: []interface{}{ + Name: "ParseUnix", + Function: ParseUnix, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "SetInStash", //FIXME: signature will probably blow everything up - function: SetInStash, - signature: []interface{}{ + Name: "SetInStash", //FIXME: signature will probably blow everything up + Function: SetInStash, + Signature: []interface{}{ new(func(string, string, string, *time.Duration) error), }, }, { - name: "Fields", - function: Fields, - signature: []interface{}{ + Name: "Fields", + Function: Fields, + Signature: []interface{}{ new(func(string) []string), }, }, { - name: "Index", - function: Index, - signature: []interface{}{ + Name: "Index", + Function: Index, + Signature: []interface{}{ new(func(string, string) int), }, }, { - name: "IndexAny", - function: IndexAny, - signature: []interface{}{ + Name: "IndexAny", + Function: IndexAny, + Signature: []interface{}{ new(func(string, string) int), }, }, { - name: "Join", - function: Join, - signature: []interface{}{ + Name: "Join", + Function: Join, + Signature: []interface{}{ new(func([]string, string) string), }, }, { - name: "Split", - function: Split, - signature: []interface{}{ + Name: "Split", + Function: Split, + Signature: []interface{}{ new(func(string, string) []string), }, }, { - name: "SplitAfter", - function: SplitAfter, - signature: []interface{}{ + Name: "SplitAfter", + Function: SplitAfter, + Signature: []interface{}{ new(func(string, string) []string), }, }, { - name: "SplitAfterN", - function: SplitAfterN, - signature: []interface{}{ + Name: "SplitAfterN", + Function: SplitAfterN, + Signature: []interface{}{ new(func(string, string, int) []string), }, }, { - name: "SplitN", - function: SplitN, - signature: []interface{}{ + Name: "SplitN", + Function: SplitN, + Signature: []interface{}{ new(func(string, string, int) []string), }, }, { - name: "Replace", - function: Replace, - signature: []interface{}{ + Name: "Replace", + Function: Replace, + Signature: []interface{}{ new(func(string, string, string, int) string), }, }, { - name: "ReplaceAll", - function: ReplaceAll, - signature: []interface{}{ + Name: "ReplaceAll", + Function: ReplaceAll, + Signature: []interface{}{ new(func(string, string, string) string), }, }, { - name: "Trim", - function: Trim, - signature: []interface{}{ + Name: "Trim", + Function: Trim, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "TrimLeft", - function: TrimLeft, - signature: []interface{}{ + Name: "TrimLeft", + Function: TrimLeft, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "TrimRight", - function: TrimRight, - signature: []interface{}{ + Name: "TrimRight", + Function: TrimRight, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "TrimSpace", - function: TrimSpace, - signature: []interface{}{ + Name: "TrimSpace", + Function: TrimSpace, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "TrimPrefix", - function: TrimPrefix, - signature: []interface{}{ + Name: "TrimPrefix", + Function: TrimPrefix, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "TrimSuffix", - function: TrimSuffix, - signature: []interface{}{ + Name: "TrimSuffix", + Function: TrimSuffix, + Signature: []interface{}{ new(func(string, string) string), }, }, { - name: "Get", - function: Get, - signature: []interface{}{ + Name: "Get", + Function: Get, + Signature: []interface{}{ new(func([]string, int) string), }, }, { - name: "ToString", - function: ToString, - signature: []interface{}{ + Name: "ToString", + Function: ToString, + Signature: []interface{}{ new(func(interface{}) string), }, }, { - name: "Match", - function: Match, - signature: []interface{}{ + Name: "Match", + Function: Match, + Signature: []interface{}{ new(func(string, string) bool), }, }, { - name: "KeyExists", - function: KeyExists, - signature: []interface{}{ + Name: "KeyExists", + Function: KeyExists, + Signature: []interface{}{ new(func(string, map[string]any) bool), }, }, { - name: "LogInfo", - function: LogInfo, - signature: []interface{}{ + Name: "LogInfo", + Function: LogInfo, + Signature: []interface{}{ new(func(string, ...interface{}) bool), }, }, { - name: "B64Decode", - function: B64Decode, - signature: []interface{}{ + Name: "B64Decode", + Function: B64Decode, + Signature: []interface{}{ new(func(string) string), }, }, { - name: "UnmarshalJSON", - function: UnmarshalJSON, - signature: []interface{}{ + Name: "UnmarshalJSON", + Function: UnmarshalJSON, + Signature: []interface{}{ new(func(string, map[string]interface{}, string) error), }, }, { - name: "ParseKV", - function: ParseKV, - signature: []interface{}{ + Name: "ParseKV", + Function: ParseKV, + Signature: []interface{}{ new(func(string, map[string]interface{}, string) error), }, }, { - name: "Hostname", - function: Hostname, - signature: []interface{}{ + Name: "Hostname", + Function: Hostname, + Signature: []interface{}{ new(func() (string, error)), }, }, { - name: "FloatApproxEqual", - function: FloatApproxEqual, - signature: []interface{}{ + Name: "FloatApproxEqual", + Function: FloatApproxEqual, + Signature: []interface{}{ new(func(float64, float64) bool), }, }, diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 79a621c7d35..350dd2974e5 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -60,9 +60,9 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option { exprFunctionOptions = []expr.Option{} for _, function := range exprFuncs { exprFunctionOptions = append(exprFunctionOptions, - expr.Function(function.name, - function.function, - function.signature..., + expr.Function(function.Name, + function.Function, + function.Signature..., )) } } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 5e792a13bbf..6029adc3eb7 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -58,12 +58,12 @@ type ReqDumpFilter struct { ArgsDrop bool } -func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { +func (r *ParsedRequest) DumpRequest(params ...any) (any, error) { filter := ReqDumpFilter{} filter.BodyDrop = true filter.HeadersNameFilters = []string{"cookie", "authorization"} filter.req = r - return &filter + return &filter, nil } // clear filters diff --git a/pkg/waf/request_test.go b/pkg/waf/request_test.go index 2625e11f5d3..b4f50761c47 100644 --- a/pkg/waf/request_test.go +++ b/pkg/waf/request_test.go @@ -161,7 +161,8 @@ func TestBodyDumper(t *testing.T) { for idx, test := range tests { t.Run(test.name, func(t *testing.T) { - orig_dr := test.req.DumpRequest() + tmp_dr, _ := test.req.DumpRequest() + orig_dr := tmp_dr.(*ReqDumpFilter) result := test.filter(orig_dr).GetFilteredRequest() if len(result.Body) != len(test.expect.Body) { diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 2820bb8c496..497d4eef7df 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -34,19 +34,26 @@ const ( func (h *Hook) Build(hookStage int) error { ctx := map[string]interface{}{} + opts := []expr.Option{} switch hookStage { case hookOnLoad: - ctx = GetOnLoadEnv(&WaapRuntimeConfig{}) + opts = GetOnLoadEnv(ctx, &WaapRuntimeConfig{}) case hookPreEval: - ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + ctx["IsInBand"] = true + ctx["IsOutBand"] = true + opts = GetPreEvalEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) case hookPostEval: - ctx = GetPostEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + ctx["IsInBand"] = true + ctx["IsOutBand"] = true + opts = GetPostEvalEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) case hookOnMatch: - ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) + ctx["evt"] = types.Event{} + ctx["IsInBand"] = true + ctx["IsOutBand"] = true + opts = GetOnMatchEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) } - opts := GetExprWAFOptions(ctx) if h.Filter != "" { - program, err := expr.Compile(h.Filter, opts...) //FIXME: opts + program, err := expr.Compile(h.Filter, opts...) if err != nil { return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err) } @@ -283,7 +290,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, map[string]interface{}{}, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_load filter %s : %w", rule.Filter, err) } @@ -299,7 +306,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, map[string]interface{}{}, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_load expr: %s", err) continue @@ -310,10 +317,14 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { - + ctx := map[string]interface{}{ + "evt": evt, + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + } for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) } @@ -329,7 +340,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_match expr: %s", err) continue @@ -340,9 +351,13 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type } func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { + ctx := map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + } for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(ctx, w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap pre_eval filter %s : %w", rule.Filter, err) } @@ -359,7 +374,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap pre_eval expr: %s", err) continue @@ -371,9 +386,13 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { } func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { + ctx := map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + } for _, rule := range w.CompiledPostEval { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap post_eval filter %s : %w", rule.Filter, err) } @@ -390,7 +409,7 @@ func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap post_eval expr: %s", err) continue @@ -551,6 +570,7 @@ func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { // func (w *WaapRuntimeConfig) SetActionByID(name string, action string) error { func (w *WaapRuntimeConfig) SetActionByName(params ...any) (any, error) { + fmt.Printf("%v+\n", w) if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index 717b1bdff46..b2a80723db8 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -1,11 +1,3 @@ package waf //This is a copy paste from expr_lib.go, we probably want to only have one ? - -type exprCustomFunc struct { - name string - function func(params ...any) (any, error) - signature []interface{} -} - -var exprFuncs = []exprCustomFunc{} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index ced1b68e787..aba4ec19b1c 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -3,40 +3,183 @@ package waf import ( "github.com/antonmedv/expr" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" ) -func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { +var exprOnLoadOptions = []expr.Option{} +var exprPreEvalOptions = []expr.Option{} +var exprPostEvalOptions = []expr.Option{} +var exprOnMatchOptions = []expr.Option{} + +func GetOnLoadEnv(ctx map[string]interface{}, w *WaapRuntimeConfig) []expr.Option { baseHelpers := exprhelpers.GetExprOptions(ctx) + onLoadHelpers := []exprhelpers.ExprCustomFunc{ + { + Name: "RemoveInBandRuleByID", + Function: w.DisableInBandRuleByID, + Signature: []interface{}{ + new(func(int) error), + }, + }, + { + Name: "RemoveInBandRuleByTag", + Function: w.DisableInBandRuleByTag, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveInBandRuleByName", + Function: w.DisableInBandRuleByName, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveOutBandRuleByID", + Function: w.DisableOutBandRuleByID, + Signature: []interface{}{ + new(func(int) error), + }, + }, + { + Name: "RemoveOutBandRuleByTag", + Function: w.DisableOutBandRuleByTag, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveOutBandRuleByName", + Function: w.DisableOutBandRuleByName, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "SetRemediationByTag", + Function: w.SetActionByTag, + Signature: []interface{}{ + new(func(string, string) error), + }, + }, + { + Name: "SetRemediationByID", + Function: w.SetActionByID, + Signature: []interface{}{ + new(func(int, string) error), + }, + }, + { + Name: "SetRemediationByName", + Function: w.SetActionByName, + Signature: []interface{}{ + new(func(string, string) error), + }, + }, + } - for _, function := range exprFuncs { - baseHelpers = append(baseHelpers, - expr.Function(function.name, - function.function, - function.signature..., - )) + if len(exprOnLoadOptions) == 0 { + for _, function := range onLoadHelpers { + exprOnLoadOptions = append(exprOnLoadOptions, + expr.Function( + function.Name, + function.Function, + function.Signature..., + ), + ) + } + exprOnLoadOptions = append(exprOnLoadOptions, baseHelpers...) } - return baseHelpers + + return exprOnLoadOptions } -func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { - //FIXME: use expr.Function instead of this - return map[string]interface{}{ - "RemoveInBandRuleByID": w.DisableInBandRuleByID, - "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, - "RemoveInBandRuleByName": w.DisableInBandRuleByName, - "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, - "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, - "RemoveOutBandRuleByName": w.DisableOutBandRuleByName, - "SetRemediationByTag": w.SetActionByTag, - "SetRemediationByID": w.SetActionByID, - "SetRemediationByName": w.SetActionByName, +func GetPreEvalEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { + + baseHelpers := exprhelpers.GetExprOptions(ctx) + preEvalHelpers := []exprhelpers.ExprCustomFunc{ + { + Name: "RemoveInBandRuleByID", + Function: w.RemoveInbandRuleByID, + Signature: []interface{}{ + new(func(int) error), + }, + }, + { + Name: "RemoveInBandRuleByTag", + Function: w.RemoveInbandRuleByTag, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveInBandRuleByName", + Function: w.RemoveInbandRuleByName, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveOutBandRuleByID", + Function: w.RemoveOutbandRuleByID, + Signature: []interface{}{ + new(func(int) error), + }, + }, + { + Name: "RemoveOutBandRuleByTag", + Function: w.RemoveOutbandRuleByTag, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "RemoveOutBandRuleByName", + Function: w.RemoveOutbandRuleByName, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "SetRemediationByTag", + Function: w.SetActionByTag, + Signature: []interface{}{ + new(func(string, string) error), + }, + }, + { + Name: "SetRemediationByID", + Function: w.SetActionByID, + Signature: []interface{}{ + new(func(int, string) error), + }, + }, + { + Name: "SetRemediationByName", + Function: w.SetActionByName, + Signature: []interface{}{ + new(func(string, string) error), + }, + }, } -} -func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { + if len(exprPreEvalOptions) == 0 { + for _, function := range preEvalHelpers { + exprPreEvalOptions = append(exprPreEvalOptions, + expr.Function( + function.Name, + function.Function, + function.Signature..., + ), + ) + } + exprPreEvalOptions = append(exprPreEvalOptions, baseHelpers...) + } + + return exprPreEvalOptions + //FIXME: use expr.Function instead of this - return map[string]interface{}{ + /*return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, "RemoveInBandRuleByID": w.RemoveInbandRuleByID, @@ -48,20 +191,114 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte "SetRemediationByTag": w.SetActionByTag, "SetRemediationByID": w.SetActionByID, "SetRemediationByName": w.SetActionByName, - } + }*/ } -func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { - //FIXME: use expr.Function instead of this +func GetPostEvalEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { + baseHelpers := exprhelpers.GetExprOptions(ctx) + postEvalHelpers := []exprhelpers.ExprCustomFunc{ + { + Name: "DumpRequest", + Function: request.DumpRequest, + Signature: []interface{}{ + new(func() *ReqDumpFilter), + }, + }, + } + + if len(exprPostEvalOptions) == 0 { + for _, function := range postEvalHelpers { + exprPostEvalOptions = append(exprPostEvalOptions, + expr.Function( + function.Name, + function.Function, + function.Signature..., + ), + ) + } + exprPostEvalOptions = append(exprPostEvalOptions, baseHelpers...) + } + + return exprPostEvalOptions + + /*//FIXME: use expr.Function instead of this return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, "DumpRequest": request.DumpRequest, - } + }*/ } -func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { - //FIXME: use expr.Function instead of this +func GetOnMatchEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { + baseHelpers := exprhelpers.GetExprOptions(ctx) + onMatchHelpers := []exprhelpers.ExprCustomFunc{ + { + Name: "SetRemediation", + Function: w.SetAction, + Signature: []interface{}{ + new(func(string) error), + }, + }, + { + Name: "SetReturnCode", + Function: w.SetHTTPCode, + Signature: []interface{}{ + new(func(int) error), + }, + }, + { + Name: "CancelEvent", + Function: w.CancelEvent, + Signature: []interface{}{ + new(func() error), + }, + }, + { + Name: "SendEvent", + Function: w.SendEvent, + Signature: []interface{}{ + new(func() error), + }, + }, + { + Name: "CancelAlert", + Function: w.CancelAlert, + Signature: []interface{}{ + new(func() error), + }, + }, + { + Name: "SendAlert", + Function: w.SendAlert, + Signature: []interface{}{ + new(func() error), + }, + }, + { + Name: "DumpRequest", + Function: request.DumpRequest, + Signature: []interface{}{ + new(func() *ReqDumpFilter), + }, + }, + } + + if len(exprOnMatchOptions) == 0 { + for _, function := range onMatchHelpers { + exprOnMatchOptions = append(exprOnMatchOptions, + expr.Function( + function.Name, + function.Function, + function.Signature..., + ), + ) + } + exprOnMatchOptions = append(exprOnMatchOptions, baseHelpers...) + } + + return exprOnMatchOptions + + /*//FIXME: use expr.Function instead of this return map[string]interface{}{ "evt": evt, "req": request, @@ -74,5 +311,5 @@ func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event "CancelAlert": w.CancelAlert, "SendAlert": w.SendAlert, "DumpRequest": request.DumpRequest, - } + }*/ } From e637e7bf8b8297df8273ab9ed704757b2e731fba Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:00:19 +0100 Subject: [PATCH 231/263] Revert "use expr func" This reverts commit ac451ccaf355b6977131f1945a946d875e268705. --- pkg/exprhelpers/expr_lib.go | 382 ++++++++++++++++++------------------ pkg/exprhelpers/helpers.go | 6 +- pkg/waf/request.go | 4 +- pkg/waf/request_test.go | 3 +- pkg/waf/waap.go | 50 ++--- pkg/waf/waf_expr_lib.go | 8 + pkg/waf/waf_helpers.go | 299 +++------------------------- 7 files changed, 251 insertions(+), 501 deletions(-) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index b4f18a8a3ab..db191b84a8d 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -6,438 +6,438 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cticlient" ) -type ExprCustomFunc struct { - Name string - Function func(params ...any) (any, error) - Signature []interface{} +type exprCustomFunc struct { + name string + function func(params ...any) (any, error) + signature []interface{} } -var exprFuncs = []ExprCustomFunc{ +var exprFuncs = []exprCustomFunc{ { - Name: "CrowdsecCTI", - Function: CrowdsecCTI, - Signature: []interface{}{ + name: "CrowdsecCTI", + function: CrowdsecCTI, + signature: []interface{}{ new(func(string) (*cticlient.SmokeItem, error)), }, }, { - Name: "Flatten", - Function: Flatten, - Signature: []interface{}{}, + name: "Flatten", + function: Flatten, + signature: []interface{}{}, }, { - Name: "Distinct", - Function: Distinct, - Signature: []interface{}{}, + name: "Distinct", + function: Distinct, + signature: []interface{}{}, }, { - Name: "FlattenDistinct", - Function: FlattenDistinct, - Signature: []interface{}{}, + name: "FlattenDistinct", + function: FlattenDistinct, + signature: []interface{}{}, }, { - Name: "Distance", - Function: Distance, - Signature: []interface{}{ + name: "Distance", + function: Distance, + signature: []interface{}{ new(func(string, string, string, string) (float64, error)), }, }, { - Name: "GetFromStash", - Function: GetFromStash, - Signature: []interface{}{ + name: "GetFromStash", + function: GetFromStash, + signature: []interface{}{ new(func(string, string) (string, error)), }, }, { - Name: "Atof", - Function: Atof, - Signature: []interface{}{ + name: "Atof", + function: Atof, + signature: []interface{}{ new(func(string) float64), }, }, { - Name: "JsonExtract", - Function: JsonExtract, - Signature: []interface{}{ + name: "JsonExtract", + function: JsonExtract, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "JsonExtractUnescape", - Function: JsonExtractUnescape, - Signature: []interface{}{ + name: "JsonExtractUnescape", + function: JsonExtractUnescape, + signature: []interface{}{ new(func(string, ...string) string), }, }, { - Name: "JsonExtractLib", - Function: JsonExtractLib, - Signature: []interface{}{ + name: "JsonExtractLib", + function: JsonExtractLib, + signature: []interface{}{ new(func(string, ...string) string), }, }, { - Name: "JsonExtractSlice", - Function: JsonExtractSlice, - Signature: []interface{}{ + name: "JsonExtractSlice", + function: JsonExtractSlice, + signature: []interface{}{ new(func(string, string) []interface{}), }, }, { - Name: "JsonExtractObject", - Function: JsonExtractObject, - Signature: []interface{}{ + name: "JsonExtractObject", + function: JsonExtractObject, + signature: []interface{}{ new(func(string, string) map[string]interface{}), }, }, { - Name: "ToJsonString", - Function: ToJson, - Signature: []interface{}{ + name: "ToJsonString", + function: ToJson, + signature: []interface{}{ new(func(interface{}) string), }, }, { - Name: "File", - Function: File, - Signature: []interface{}{ + name: "File", + function: File, + signature: []interface{}{ new(func(string) []string), }, }, { - Name: "RegexpInFile", - Function: RegexpInFile, - Signature: []interface{}{ + name: "RegexpInFile", + function: RegexpInFile, + signature: []interface{}{ new(func(string, string) bool), }, }, { - Name: "Upper", - Function: Upper, - Signature: []interface{}{ + name: "Upper", + function: Upper, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "Lower", - Function: Lower, - Signature: []interface{}{ + name: "Lower", + function: Lower, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "IpInRange", - Function: IpInRange, - Signature: []interface{}{ + name: "IpInRange", + function: IpInRange, + signature: []interface{}{ new(func(string, string) bool), }, }, { - Name: "TimeNow", - Function: TimeNow, - Signature: []interface{}{ + name: "TimeNow", + function: TimeNow, + signature: []interface{}{ new(func() string), }, }, { - Name: "ParseUri", - Function: ParseUri, - Signature: []interface{}{ + name: "ParseUri", + function: ParseUri, + signature: []interface{}{ new(func(string) map[string][]string), }, }, { - Name: "PathUnescape", - Function: PathUnescape, - Signature: []interface{}{ + name: "PathUnescape", + function: PathUnescape, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "QueryUnescape", - Function: QueryUnescape, - Signature: []interface{}{ + name: "QueryUnescape", + function: QueryUnescape, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "PathEscape", - Function: PathEscape, - Signature: []interface{}{ + name: "PathEscape", + function: PathEscape, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "QueryEscape", - Function: QueryEscape, - Signature: []interface{}{ + name: "QueryEscape", + function: QueryEscape, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "XMLGetAttributeValue", - Function: XMLGetAttributeValue, - Signature: []interface{}{ + name: "XMLGetAttributeValue", + function: XMLGetAttributeValue, + signature: []interface{}{ new(func(string, string, string) string), }, }, { - Name: "XMLGetNodeValue", - Function: XMLGetNodeValue, - Signature: []interface{}{ + name: "XMLGetNodeValue", + function: XMLGetNodeValue, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "IpToRange", - Function: IpToRange, - Signature: []interface{}{ + name: "IpToRange", + function: IpToRange, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "IsIPV6", - Function: IsIPV6, - Signature: []interface{}{ + name: "IsIPV6", + function: IsIPV6, + signature: []interface{}{ new(func(string) bool), }, }, { - Name: "IsIPV4", - Function: IsIPV4, - Signature: []interface{}{ + name: "IsIPV4", + function: IsIPV4, + signature: []interface{}{ new(func(string) bool), }, }, { - Name: "IsIP", - Function: IsIP, - Signature: []interface{}{ + name: "IsIP", + function: IsIP, + signature: []interface{}{ new(func(string) bool), }, }, { - Name: "LookupHost", - Function: LookupHost, - Signature: []interface{}{ + name: "LookupHost", + function: LookupHost, + signature: []interface{}{ new(func(string) []string), }, }, { - Name: "GetDecisionsCount", - Function: GetDecisionsCount, - Signature: []interface{}{ + name: "GetDecisionsCount", + function: GetDecisionsCount, + signature: []interface{}{ new(func(string) int), }, }, { - Name: "GetDecisionsSinceCount", - Function: GetDecisionsSinceCount, - Signature: []interface{}{ + name: "GetDecisionsSinceCount", + function: GetDecisionsSinceCount, + signature: []interface{}{ new(func(string, string) int), }, }, { - Name: "Sprintf", - Function: Sprintf, - Signature: []interface{}{ + name: "Sprintf", + function: Sprintf, + signature: []interface{}{ new(func(string, ...interface{}) string), }, }, { - Name: "ParseUnix", - Function: ParseUnix, - Signature: []interface{}{ + name: "ParseUnix", + function: ParseUnix, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "SetInStash", //FIXME: signature will probably blow everything up - Function: SetInStash, - Signature: []interface{}{ + name: "SetInStash", //FIXME: signature will probably blow everything up + function: SetInStash, + signature: []interface{}{ new(func(string, string, string, *time.Duration) error), }, }, { - Name: "Fields", - Function: Fields, - Signature: []interface{}{ + name: "Fields", + function: Fields, + signature: []interface{}{ new(func(string) []string), }, }, { - Name: "Index", - Function: Index, - Signature: []interface{}{ + name: "Index", + function: Index, + signature: []interface{}{ new(func(string, string) int), }, }, { - Name: "IndexAny", - Function: IndexAny, - Signature: []interface{}{ + name: "IndexAny", + function: IndexAny, + signature: []interface{}{ new(func(string, string) int), }, }, { - Name: "Join", - Function: Join, - Signature: []interface{}{ + name: "Join", + function: Join, + signature: []interface{}{ new(func([]string, string) string), }, }, { - Name: "Split", - Function: Split, - Signature: []interface{}{ + name: "Split", + function: Split, + signature: []interface{}{ new(func(string, string) []string), }, }, { - Name: "SplitAfter", - Function: SplitAfter, - Signature: []interface{}{ + name: "SplitAfter", + function: SplitAfter, + signature: []interface{}{ new(func(string, string) []string), }, }, { - Name: "SplitAfterN", - Function: SplitAfterN, - Signature: []interface{}{ + name: "SplitAfterN", + function: SplitAfterN, + signature: []interface{}{ new(func(string, string, int) []string), }, }, { - Name: "SplitN", - Function: SplitN, - Signature: []interface{}{ + name: "SplitN", + function: SplitN, + signature: []interface{}{ new(func(string, string, int) []string), }, }, { - Name: "Replace", - Function: Replace, - Signature: []interface{}{ + name: "Replace", + function: Replace, + signature: []interface{}{ new(func(string, string, string, int) string), }, }, { - Name: "ReplaceAll", - Function: ReplaceAll, - Signature: []interface{}{ + name: "ReplaceAll", + function: ReplaceAll, + signature: []interface{}{ new(func(string, string, string) string), }, }, { - Name: "Trim", - Function: Trim, - Signature: []interface{}{ + name: "Trim", + function: Trim, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "TrimLeft", - Function: TrimLeft, - Signature: []interface{}{ + name: "TrimLeft", + function: TrimLeft, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "TrimRight", - Function: TrimRight, - Signature: []interface{}{ + name: "TrimRight", + function: TrimRight, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "TrimSpace", - Function: TrimSpace, - Signature: []interface{}{ + name: "TrimSpace", + function: TrimSpace, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "TrimPrefix", - Function: TrimPrefix, - Signature: []interface{}{ + name: "TrimPrefix", + function: TrimPrefix, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "TrimSuffix", - Function: TrimSuffix, - Signature: []interface{}{ + name: "TrimSuffix", + function: TrimSuffix, + signature: []interface{}{ new(func(string, string) string), }, }, { - Name: "Get", - Function: Get, - Signature: []interface{}{ + name: "Get", + function: Get, + signature: []interface{}{ new(func([]string, int) string), }, }, { - Name: "ToString", - Function: ToString, - Signature: []interface{}{ + name: "ToString", + function: ToString, + signature: []interface{}{ new(func(interface{}) string), }, }, { - Name: "Match", - Function: Match, - Signature: []interface{}{ + name: "Match", + function: Match, + signature: []interface{}{ new(func(string, string) bool), }, }, { - Name: "KeyExists", - Function: KeyExists, - Signature: []interface{}{ + name: "KeyExists", + function: KeyExists, + signature: []interface{}{ new(func(string, map[string]any) bool), }, }, { - Name: "LogInfo", - Function: LogInfo, - Signature: []interface{}{ + name: "LogInfo", + function: LogInfo, + signature: []interface{}{ new(func(string, ...interface{}) bool), }, }, { - Name: "B64Decode", - Function: B64Decode, - Signature: []interface{}{ + name: "B64Decode", + function: B64Decode, + signature: []interface{}{ new(func(string) string), }, }, { - Name: "UnmarshalJSON", - Function: UnmarshalJSON, - Signature: []interface{}{ + name: "UnmarshalJSON", + function: UnmarshalJSON, + signature: []interface{}{ new(func(string, map[string]interface{}, string) error), }, }, { - Name: "ParseKV", - Function: ParseKV, - Signature: []interface{}{ + name: "ParseKV", + function: ParseKV, + signature: []interface{}{ new(func(string, map[string]interface{}, string) error), }, }, { - Name: "Hostname", - Function: Hostname, - Signature: []interface{}{ + name: "Hostname", + function: Hostname, + signature: []interface{}{ new(func() (string, error)), }, }, { - Name: "FloatApproxEqual", - Function: FloatApproxEqual, - Signature: []interface{}{ + name: "FloatApproxEqual", + function: FloatApproxEqual, + signature: []interface{}{ new(func(float64, float64) bool), }, }, diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 350dd2974e5..79a621c7d35 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -60,9 +60,9 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option { exprFunctionOptions = []expr.Option{} for _, function := range exprFuncs { exprFunctionOptions = append(exprFunctionOptions, - expr.Function(function.Name, - function.Function, - function.Signature..., + expr.Function(function.name, + function.function, + function.signature..., )) } } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 6029adc3eb7..5e792a13bbf 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -58,12 +58,12 @@ type ReqDumpFilter struct { ArgsDrop bool } -func (r *ParsedRequest) DumpRequest(params ...any) (any, error) { +func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { filter := ReqDumpFilter{} filter.BodyDrop = true filter.HeadersNameFilters = []string{"cookie", "authorization"} filter.req = r - return &filter, nil + return &filter } // clear filters diff --git a/pkg/waf/request_test.go b/pkg/waf/request_test.go index b4f50761c47..2625e11f5d3 100644 --- a/pkg/waf/request_test.go +++ b/pkg/waf/request_test.go @@ -161,8 +161,7 @@ func TestBodyDumper(t *testing.T) { for idx, test := range tests { t.Run(test.name, func(t *testing.T) { - tmp_dr, _ := test.req.DumpRequest() - orig_dr := tmp_dr.(*ReqDumpFilter) + orig_dr := test.req.DumpRequest() result := test.filter(orig_dr).GetFilteredRequest() if len(result.Body) != len(test.expect.Body) { diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 497d4eef7df..2820bb8c496 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -34,26 +34,19 @@ const ( func (h *Hook) Build(hookStage int) error { ctx := map[string]interface{}{} - opts := []expr.Option{} switch hookStage { case hookOnLoad: - opts = GetOnLoadEnv(ctx, &WaapRuntimeConfig{}) + ctx = GetOnLoadEnv(&WaapRuntimeConfig{}) case hookPreEval: - ctx["IsInBand"] = true - ctx["IsOutBand"] = true - opts = GetPreEvalEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) case hookPostEval: - ctx["IsInBand"] = true - ctx["IsOutBand"] = true - opts = GetPostEvalEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetPostEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) case hookOnMatch: - ctx["evt"] = types.Event{} - ctx["IsInBand"] = true - ctx["IsOutBand"] = true - opts = GetOnMatchEnv(ctx, &WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) } + opts := GetExprWAFOptions(ctx) if h.Filter != "" { - program, err := expr.Compile(h.Filter, opts...) + program, err := expr.Compile(h.Filter, opts...) //FIXME: opts if err != nil { return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err) } @@ -290,7 +283,7 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, map[string]interface{}{}, w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_load filter %s : %w", rule.Filter, err) } @@ -306,7 +299,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, map[string]interface{}{}, w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_load expr: %s", err) continue @@ -317,14 +310,10 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { } func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { - ctx := map[string]interface{}{ - "evt": evt, - "IsInBand": request.IsInBand, - "IsOutBand": request.IsOutBand, - } + for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) } @@ -340,7 +329,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap on_match expr: %s", err) continue @@ -351,13 +340,9 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type } func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { - ctx := map[string]interface{}{ - "IsInBand": request.IsInBand, - "IsOutBand": request.IsOutBand, - } for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(ctx, w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap pre_eval filter %s : %w", rule.Filter, err) } @@ -374,7 +359,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap pre_eval expr: %s", err) continue @@ -386,13 +371,9 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { } func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { - ctx := map[string]interface{}{ - "IsInBand": request.IsInBand, - "IsOutBand": request.IsOutBand, - } for _, rule := range w.CompiledPostEval { if rule.FilterExpr != nil { - output, err := exprhelpers.Run(rule.FilterExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) + output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { return fmt.Errorf("unable to run waap post_eval filter %s : %w", rule.Filter, err) } @@ -409,7 +390,7 @@ func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, ctx, w.Logger, w.Logger.Level >= log.DebugLevel) + _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { log.Errorf("unable to apply waap post_eval expr: %s", err) continue @@ -570,7 +551,6 @@ func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { // func (w *WaapRuntimeConfig) SetActionByID(name string, action string) error { func (w *WaapRuntimeConfig) SetActionByName(params ...any) (any, error) { - fmt.Printf("%v+\n", w) if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go index b2a80723db8..717b1bdff46 100644 --- a/pkg/waf/waf_expr_lib.go +++ b/pkg/waf/waf_expr_lib.go @@ -1,3 +1,11 @@ package waf //This is a copy paste from expr_lib.go, we probably want to only have one ? + +type exprCustomFunc struct { + name string + function func(params ...any) (any, error) + signature []interface{} +} + +var exprFuncs = []exprCustomFunc{} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index aba4ec19b1c..ced1b68e787 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -3,183 +3,40 @@ package waf import ( "github.com/antonmedv/expr" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) -var exprOnLoadOptions = []expr.Option{} -var exprPreEvalOptions = []expr.Option{} -var exprPostEvalOptions = []expr.Option{} -var exprOnMatchOptions = []expr.Option{} - -func GetOnLoadEnv(ctx map[string]interface{}, w *WaapRuntimeConfig) []expr.Option { +func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { baseHelpers := exprhelpers.GetExprOptions(ctx) - onLoadHelpers := []exprhelpers.ExprCustomFunc{ - { - Name: "RemoveInBandRuleByID", - Function: w.DisableInBandRuleByID, - Signature: []interface{}{ - new(func(int) error), - }, - }, - { - Name: "RemoveInBandRuleByTag", - Function: w.DisableInBandRuleByTag, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveInBandRuleByName", - Function: w.DisableInBandRuleByName, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveOutBandRuleByID", - Function: w.DisableOutBandRuleByID, - Signature: []interface{}{ - new(func(int) error), - }, - }, - { - Name: "RemoveOutBandRuleByTag", - Function: w.DisableOutBandRuleByTag, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveOutBandRuleByName", - Function: w.DisableOutBandRuleByName, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "SetRemediationByTag", - Function: w.SetActionByTag, - Signature: []interface{}{ - new(func(string, string) error), - }, - }, - { - Name: "SetRemediationByID", - Function: w.SetActionByID, - Signature: []interface{}{ - new(func(int, string) error), - }, - }, - { - Name: "SetRemediationByName", - Function: w.SetActionByName, - Signature: []interface{}{ - new(func(string, string) error), - }, - }, - } - if len(exprOnLoadOptions) == 0 { - for _, function := range onLoadHelpers { - exprOnLoadOptions = append(exprOnLoadOptions, - expr.Function( - function.Name, - function.Function, - function.Signature..., - ), - ) - } - exprOnLoadOptions = append(exprOnLoadOptions, baseHelpers...) + for _, function := range exprFuncs { + baseHelpers = append(baseHelpers, + expr.Function(function.name, + function.function, + function.signature..., + )) } - - return exprOnLoadOptions + return baseHelpers } -func GetPreEvalEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { - - baseHelpers := exprhelpers.GetExprOptions(ctx) - preEvalHelpers := []exprhelpers.ExprCustomFunc{ - { - Name: "RemoveInBandRuleByID", - Function: w.RemoveInbandRuleByID, - Signature: []interface{}{ - new(func(int) error), - }, - }, - { - Name: "RemoveInBandRuleByTag", - Function: w.RemoveInbandRuleByTag, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveInBandRuleByName", - Function: w.RemoveInbandRuleByName, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveOutBandRuleByID", - Function: w.RemoveOutbandRuleByID, - Signature: []interface{}{ - new(func(int) error), - }, - }, - { - Name: "RemoveOutBandRuleByTag", - Function: w.RemoveOutbandRuleByTag, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "RemoveOutBandRuleByName", - Function: w.RemoveOutbandRuleByName, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "SetRemediationByTag", - Function: w.SetActionByTag, - Signature: []interface{}{ - new(func(string, string) error), - }, - }, - { - Name: "SetRemediationByID", - Function: w.SetActionByID, - Signature: []interface{}{ - new(func(int, string) error), - }, - }, - { - Name: "SetRemediationByName", - Function: w.SetActionByName, - Signature: []interface{}{ - new(func(string, string) error), - }, - }, - } - - if len(exprPreEvalOptions) == 0 { - for _, function := range preEvalHelpers { - exprPreEvalOptions = append(exprPreEvalOptions, - expr.Function( - function.Name, - function.Function, - function.Signature..., - ), - ) - } - exprPreEvalOptions = append(exprPreEvalOptions, baseHelpers...) +func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { + //FIXME: use expr.Function instead of this + return map[string]interface{}{ + "RemoveInBandRuleByID": w.DisableInBandRuleByID, + "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, + "RemoveInBandRuleByName": w.DisableInBandRuleByName, + "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, + "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, + "RemoveOutBandRuleByName": w.DisableOutBandRuleByName, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, } +} - return exprPreEvalOptions - +func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { //FIXME: use expr.Function instead of this - /*return map[string]interface{}{ + return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, "RemoveInBandRuleByID": w.RemoveInbandRuleByID, @@ -191,114 +48,20 @@ func GetPreEvalEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *Pa "SetRemediationByTag": w.SetActionByTag, "SetRemediationByID": w.SetActionByID, "SetRemediationByName": w.SetActionByName, - }*/ -} - -func GetPostEvalEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { - baseHelpers := exprhelpers.GetExprOptions(ctx) - postEvalHelpers := []exprhelpers.ExprCustomFunc{ - { - Name: "DumpRequest", - Function: request.DumpRequest, - Signature: []interface{}{ - new(func() *ReqDumpFilter), - }, - }, - } - - if len(exprPostEvalOptions) == 0 { - for _, function := range postEvalHelpers { - exprPostEvalOptions = append(exprPostEvalOptions, - expr.Function( - function.Name, - function.Function, - function.Signature..., - ), - ) - } - exprPostEvalOptions = append(exprPostEvalOptions, baseHelpers...) } +} - return exprPostEvalOptions - - /*//FIXME: use expr.Function instead of this +func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { + //FIXME: use expr.Function instead of this return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, "DumpRequest": request.DumpRequest, - }*/ -} - -func GetOnMatchEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *ParsedRequest) []expr.Option { - baseHelpers := exprhelpers.GetExprOptions(ctx) - onMatchHelpers := []exprhelpers.ExprCustomFunc{ - { - Name: "SetRemediation", - Function: w.SetAction, - Signature: []interface{}{ - new(func(string) error), - }, - }, - { - Name: "SetReturnCode", - Function: w.SetHTTPCode, - Signature: []interface{}{ - new(func(int) error), - }, - }, - { - Name: "CancelEvent", - Function: w.CancelEvent, - Signature: []interface{}{ - new(func() error), - }, - }, - { - Name: "SendEvent", - Function: w.SendEvent, - Signature: []interface{}{ - new(func() error), - }, - }, - { - Name: "CancelAlert", - Function: w.CancelAlert, - Signature: []interface{}{ - new(func() error), - }, - }, - { - Name: "SendAlert", - Function: w.SendAlert, - Signature: []interface{}{ - new(func() error), - }, - }, - { - Name: "DumpRequest", - Function: request.DumpRequest, - Signature: []interface{}{ - new(func() *ReqDumpFilter), - }, - }, - } - - if len(exprOnMatchOptions) == 0 { - for _, function := range onMatchHelpers { - exprOnMatchOptions = append(exprOnMatchOptions, - expr.Function( - function.Name, - function.Function, - function.Signature..., - ), - ) - } - exprOnMatchOptions = append(exprOnMatchOptions, baseHelpers...) } +} - return exprOnMatchOptions - - /*//FIXME: use expr.Function instead of this +func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { + //FIXME: use expr.Function instead of this return map[string]interface{}{ "evt": evt, "req": request, @@ -311,5 +74,5 @@ func GetOnMatchEnv(ctx map[string]interface{}, w *WaapRuntimeConfig, request *Pa "CancelAlert": w.CancelAlert, "SendAlert": w.SendAlert, "DumpRequest": request.DumpRequest, - }*/ + } } From 1c22783661d18f321c5ded3809f3d05bbd4f185d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:16:01 +0100 Subject: [PATCH 232/263] no need for any in helpers as we are not using expr.Function --- pkg/acquisition/modules/waap/waap.go | 2 - pkg/waf/waap.go | 149 ++++++++++----------------- pkg/waf/waf_expr_lib.go | 11 -- pkg/waf/waf_helpers.go | 19 ---- 4 files changed, 53 insertions(+), 128 deletions(-) delete mode 100644 pkg/waf/waf_expr_lib.go diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 8830ef993cb..178a53adaeb 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -364,8 +364,6 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { WafBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() } - w.logger.Infof("Response: %+v", response) - waapResponse := w.WaapRuntime.GenerateResponse(response) rw.WriteHeader(waapResponse.HTTPStatus) diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 2820bb8c496..228f904c91e 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -44,7 +44,7 @@ func (h *Hook) Build(hookStage int) error { case hookOnMatch: ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) } - opts := GetExprWAFOptions(ctx) + opts := exprhelpers.GetExprOptions(ctx) if h.Filter != "" { program, err := expr.Compile(h.Filter, opts...) //FIXME: opts if err != nil { @@ -401,170 +401,130 @@ func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { return nil } -/* @sbl / @tko -add the helpers to: - - remove by id-range - - remove by tag - - set remediation by tag/id-range - -*/ - -// func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { -func (w *WaapRuntimeConfig) RemoveInbandRuleByID(params ...any) (any, error) { - id := params[0].(int) +func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { w.Logger.Debugf("removing inband rule %d", id) - _ = w.InBandTx.RemoveRuleByIDWithError(id) - return nil, nil + return w.InBandTx.RemoveRuleByIDWithError(id) } -// func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { -func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(params ...any) (any, error) { - id := params[0].(int) +func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { w.Logger.Debugf("removing outband rule %d", id) - _ = w.OutOfBandTx.RemoveRuleByIDWithError(id) - return nil, nil + return w.OutOfBandTx.RemoveRuleByIDWithError(id) } -// func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(tag string) error { -func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(params ...any) (any, error) { - tag := params[0].(string) +func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(tag string) error { w.Logger.Debugf("removing inband rule with tag %s", tag) - _ = w.InBandTx.RemoveRuleByTagWithError(tag) - return nil, nil + return w.InBandTx.RemoveRuleByTagWithError(tag) } -// func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { -func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(params ...any) (any, error) { - tag := params[0].(string) +func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { w.Logger.Debugf("removing outband rule with tag %s", tag) - _ = w.OutOfBandTx.RemoveRuleByTagWithError(tag) - return nil, nil + return w.OutOfBandTx.RemoveRuleByTagWithError(tag) } -// func (w *WaapRuntimeConfig) RemoveInbandRuleByName(name string) error { -func (w *WaapRuntimeConfig) RemoveInbandRuleByName(params ...any) (any, error) { - tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) +func (w *WaapRuntimeConfig) RemoveInbandRuleByName(name string) error { + tag := fmt.Sprintf("crowdsec-%s", name) w.Logger.Debugf("removing inband rule %s", tag) - _ = w.InBandTx.RemoveRuleByTagWithError(tag) - return nil, nil + return w.InBandTx.RemoveRuleByTagWithError(tag) } -// func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(name string) error { -func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(params ...any) (any, error) { - tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) +func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(name string) error { + tag := fmt.Sprintf("crowdsec-%s", name) w.Logger.Debugf("removing outband rule %s", tag) - _ = w.OutOfBandTx.RemoveRuleByTagWithError(tag) - return nil, nil + return w.OutOfBandTx.RemoveRuleByTagWithError(tag) } -func (w *WaapRuntimeConfig) CancelEvent(params ...any) (any, error) { +func (w *WaapRuntimeConfig) CancelEvent() error { w.Logger.Debugf("canceling event") w.Response.SendEvent = false - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByID(params ...any) (any, error) { - w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, params[0].(int)) - return nil, nil +func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { + w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, id) + return nil } -// func (w *WaapRuntimeConfig) DisableInBandRuleByName(name string) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByName(params ...any) (any, error) { - tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) +func (w *WaapRuntimeConfig) DisableInBandRuleByName(name string) error { + tagValue := fmt.Sprintf("crowdsec-%s", name) w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tagValue) - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) DisableInBandRuleByTag(tag string) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByTag(params ...any) (any, error) { - w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, params[0].(string)) - return nil, nil +func (w *WaapRuntimeConfig) DisableInBandRuleByTag(tag string) error { + w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tag) + return nil } -// func (w *WaapRuntimeConfig) DisableOutBandRuleByID(id int) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByID(params ...any) (any, error) { - w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, params[0].(int)) - return nil, nil +func (w *WaapRuntimeConfig) DisableOutBandRuleByID(id int) error { + w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, id) + return nil } -// func (w *WaapRuntimeConfig) DisableOutBandRuleByName(name string) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByName(params ...any) (any, error) { - tagValue := fmt.Sprintf("crowdsec-%s", params[0].(string)) +func (w *WaapRuntimeConfig) DisableOutBandRuleByName(name string) error { + tagValue := fmt.Sprintf("crowdsec-%s", name) w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tagValue) - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(params ...any) (any, error) { - w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, params[0].(string)) - return nil, nil +func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { + w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tag) + return nil } -func (w *WaapRuntimeConfig) SendEvent(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SendEvent() error { w.Logger.Debugf("sending event") w.Response.SendEvent = true - return nil, nil + return nil } -func (w *WaapRuntimeConfig) SendAlert(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SendAlert() error { w.Logger.Debugf("sending alert") w.Response.SendAlert = true - return nil, nil + return nil } -func (w *WaapRuntimeConfig) CancelAlert(params ...any) (any, error) { +func (w *WaapRuntimeConfig) CancelAlert() error { w.Logger.Debugf("canceling alert") w.Response.SendAlert = false - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { -func (w *WaapRuntimeConfig) SetActionByTag(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } - tag := params[0].(string) - action := params[1].(string) w.Logger.Debugf("setting action of %s to %s", tag, action) w.RemediationByTag[tag] = action - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { -func (w *WaapRuntimeConfig) SetActionByID(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { if w.RemediationById == nil { w.RemediationById = make(map[int]string) } - id := params[0].(int) - action := params[1].(string) w.Logger.Debugf("setting action of %d to %s", id, action) w.RemediationById[id] = action - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) SetActionByID(name string, action string) error { -func (w *WaapRuntimeConfig) SetActionByName(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SetActionByName(name string, action string) error { if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } - tag := fmt.Sprintf("crowdsec-%s", params[0].(string)) - action := params[1].(string) + tag := fmt.Sprintf("crowdsec-%s", name) w.Logger.Debugf("setting action of %s to %s", tag, action) w.RemediationByTag[tag] = action - return nil, nil + return nil } -// func (w *WaapRuntimeConfig) SetAction(action string) error { -func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { +func (w *WaapRuntimeConfig) SetAction(action string) error { //log.Infof("setting to %s", action) - action := params[0].(string) w.Logger.Debugf("setting action to %s", action) switch action { case "allow": @@ -579,18 +539,15 @@ func (w *WaapRuntimeConfig) SetAction(params ...any) (any, error) { case "captcha": w.Response.Action = action default: - return nil, fmt.Errorf("unknown action %s", action) + return fmt.Errorf("unknown action %s", action) } - return nil, nil - + return nil } -// func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { -func (w *WaapRuntimeConfig) SetHTTPCode(params ...any) (any, error) { - code := params[0].(int) +func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { w.Logger.Debugf("setting http code to %d", code) w.Response.HTTPResponseCode = code - return nil, nil + return nil } type BodyResponse struct { diff --git a/pkg/waf/waf_expr_lib.go b/pkg/waf/waf_expr_lib.go deleted file mode 100644 index 717b1bdff46..00000000000 --- a/pkg/waf/waf_expr_lib.go +++ /dev/null @@ -1,11 +0,0 @@ -package waf - -//This is a copy paste from expr_lib.go, we probably want to only have one ? - -type exprCustomFunc struct { - name string - function func(params ...any) (any, error) - signature []interface{} -} - -var exprFuncs = []exprCustomFunc{} diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index ced1b68e787..a978648778e 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -1,26 +1,10 @@ package waf import ( - "github.com/antonmedv/expr" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) -func GetExprWAFOptions(ctx map[string]interface{}) []expr.Option { - baseHelpers := exprhelpers.GetExprOptions(ctx) - - for _, function := range exprFuncs { - baseHelpers = append(baseHelpers, - expr.Function(function.name, - function.function, - function.signature..., - )) - } - return baseHelpers -} - func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { - //FIXME: use expr.Function instead of this return map[string]interface{}{ "RemoveInBandRuleByID": w.DisableInBandRuleByID, "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, @@ -35,7 +19,6 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { } func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { - //FIXME: use expr.Function instead of this return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, @@ -52,7 +35,6 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte } func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { - //FIXME: use expr.Function instead of this return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, @@ -61,7 +43,6 @@ func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]int } func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { - //FIXME: use expr.Function instead of this return map[string]interface{}{ "evt": evt, "req": request, From 42e1da2507ebd5c7ea7cad3855d3580947cc1e45 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:18:48 +0100 Subject: [PATCH 233/263] merge listen_addr and listen_port, default to 127.0.0.1:7442 if not set --- pkg/acquisition/modules/waap/waap.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/waap/waap.go index 178a53adaeb..72e137216bb 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/waap/waap.go @@ -34,7 +34,6 @@ var ( // configuration structure of the acquis for the Waap type WaapSourceConfig struct { ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` CertFilePath string `yaml:"cert_file"` KeyFilePath string `yaml:"key_file"` Path string `yaml:"path"` @@ -105,11 +104,7 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { wc.config.LogLevel = level } if wc.config.ListenAddr == "" { - return fmt.Errorf("listen_addr cannot be empty") - } - - if wc.config.ListenPort == 0 { - return fmt.Errorf("listen_port cannot be empty") + wc.config.ListenAddr = "127.0.0.1:7422" } if wc.config.Path == "" { @@ -134,7 +129,7 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { } if wc.config.Name == "" { - wc.config.Name = fmt.Sprintf("%s:%d%s", wc.config.ListenAddr, wc.config.ListenPort, wc.config.Path) + wc.config.Name = fmt.Sprintf("%s%s", wc.config.ListenAddr, wc.config.Path) } csConfig := csconfig.GetConfig() @@ -167,12 +162,10 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { w.logger.Infof("Cache duration for auth not set, using default: %v", *w.config.AuthCacheDuration) } - w.addr = fmt.Sprintf("%s:%d", w.config.ListenAddr, w.config.ListenPort) - w.mux = http.NewServeMux() w.server = &http.Server{ - Addr: w.addr, + Addr: w.config.ListenAddr, Handler: w.mux, } @@ -266,7 +259,7 @@ func (w *WaapSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er }) } - w.logger.Infof("Starting WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + w.logger.Infof("Starting WAF server on %s%s", w.config.ListenAddr, w.config.Path) t.Go(func() error { var err error if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { @@ -281,7 +274,7 @@ func (w *WaapSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er return nil }) <-t.Dying() - w.logger.Infof("Stopping WAF server on %s:%d%s", w.config.ListenAddr, w.config.ListenPort, w.config.Path) + w.logger.Infof("Stopping WAF server on %s%s", w.config.ListenAddr, w.config.Path) w.server.Shutdown(context.TODO()) return nil }) From c3a4066646dfcde90627b01fca9dff42cd666d76 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:41:51 +0100 Subject: [PATCH 234/263] appsec renaming, part 1 --- cmd/crowdsec/metrics.go | 8 +- cmd/crowdsec/parse.go | 2 +- pkg/acquisition/acquisition.go | 4 +- .../modules/{waap => appsec}/README.md | 0 .../{waap/waap.go => appsec/appsec.go} | 145 +++++++++--------- .../appsec_runner.go} | 34 ++-- pkg/acquisition/modules/appsec/metrics.go | 54 +++++++ .../modules/{waap => appsec}/rx_operator.go | 2 +- .../modules/{waap => appsec}/utils.go | 24 +-- pkg/acquisition/modules/waap/metrics.go | 54 ------- pkg/types/{waap_event.go => appsec_event.go} | 4 +- pkg/types/event.go | 4 +- 12 files changed, 167 insertions(+), 168 deletions(-) rename pkg/acquisition/modules/{waap => appsec}/README.md (100%) rename pkg/acquisition/modules/{waap/waap.go => appsec/appsec.go} (61%) rename pkg/acquisition/modules/{waap/waap_runner.go => appsec/appsec_runner.go} (89%) create mode 100644 pkg/acquisition/modules/appsec/metrics.go rename pkg/acquisition/modules/{waap => appsec}/rx_operator.go (98%) rename pkg/acquisition/modules/{waap => appsec}/utils.go (89%) delete mode 100644 pkg/acquisition/modules/waap/metrics.go rename pkg/types/{waap_event.go => appsec_event.go} (98%) diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 33b36b851b5..c9d88501a67 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -12,7 +12,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/go-cs-lib/version" - waap "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waap" + appsec "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -163,8 +163,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, leaky.BucketsCurrentCount, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - waap.WafGlobalParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, - waap.WafBlockCounter, + appsec.AppsecGlobalParsingHistogram, appsec.AppsecReqCounter, appsec.AppsecRuleHits, + appsec.AppsecBlockCounter, ) } else { log.Infof("Loading prometheus collectors") @@ -175,7 +175,7 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - waap.WafGlobalParsingHistogram, waap.WafInbandParsingHistogram, waap.WafOutbandParsingHistogram, waap.WafReqCounter, waap.WafRuleHits, waap.WafBlockCounter, + appsec.AppsecGlobalParsingHistogram, appsec.AppsecInbandParsingHistogram, appsec.AppsecOutbandParsingHistogram, appsec.AppsecReqCounter, appsec.AppsecRuleHits, appsec.AppsecBlockCounter, ) } diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index 13ae216fc52..e1945b027dd 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -25,7 +25,7 @@ LOOP: /*Waap is going to generate 2 events: - one that is treated as a log and can go to scenarios - another one that will go directly to LAPI*/ - if event.Type == types.WAAP { + if event.Type == types.APPSEC { outputEventChan <- event continue } diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 0e377a5cc2c..960de824144 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" @@ -28,7 +29,6 @@ import ( lokiacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki" s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3" syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog" - wafacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waap" wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" @@ -77,7 +77,7 @@ var AcquisitionSources = map[string]func() DataSource{ "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "loki": func() DataSource { return &lokiacquisition.LokiSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, - "waf": func() DataSource { return &wafacquisition.WaapSource{} }, + "waf": func() DataSource { return &appsecacquisition.AppsecSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/modules/waap/README.md b/pkg/acquisition/modules/appsec/README.md similarity index 100% rename from pkg/acquisition/modules/waap/README.md rename to pkg/acquisition/modules/appsec/README.md diff --git a/pkg/acquisition/modules/waap/waap.go b/pkg/acquisition/modules/appsec/appsec.go similarity index 61% rename from pkg/acquisition/modules/waap/waap.go rename to pkg/acquisition/modules/appsec/appsec.go index 72e137216bb..0c147f932de 100644 --- a/pkg/acquisition/modules/waap/waap.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -1,4 +1,4 @@ -package wafacquisition +package appsecacquisition import ( "context" @@ -32,32 +32,32 @@ var ( ) // configuration structure of the acquis for the Waap -type WaapSourceConfig struct { +type AppsecSourceConfig struct { ListenAddr string `yaml:"listen_addr"` CertFilePath string `yaml:"cert_file"` KeyFilePath string `yaml:"key_file"` Path string `yaml:"path"` Routines int `yaml:"routines"` - WaapConfig string `yaml:"waap_config"` - WaapConfigPath string `yaml:"waap_config_path"` + AppsecConfig string `yaml:"appsec_config"` + AppsecConfigPath string `yaml:"appsec_config_path"` AuthCacheDuration *time.Duration `yaml:"auth_cache_duration"` configuration.DataSourceCommonCfg `yaml:",inline"` } // runtime structure of WaapSourceConfig -type WaapSource struct { - config WaapSourceConfig - logger *log.Entry - mux *http.ServeMux - server *http.Server - addr string - outChan chan types.Event - InChan chan waf.ParsedRequest - WaapRuntime *waf.WaapRuntimeConfig - WaapConfigs map[string]waf.WaapConfig - lapiURL string - AuthCache AuthCache - WaapRunners []WaapRunner //one for each go-routine +type AppsecSource struct { + config AppsecSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + addr string + outChan chan types.Event + InChan chan waf.ParsedRequest + AppsecRuntime *waf.WaapRuntimeConfig + AppsecConfigs map[string]waf.WaapConfig + lapiURL string + AuthCache AuthCache + AppsecRunners []AppsecRunner //one for each go-routine } // Struct to handle cache of authentication @@ -91,11 +91,11 @@ type BodyResponse struct { Action string `json:"action"` } -func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { +func (wc *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &wc.config) if err != nil { - return errors.Wrap(err, "Cannot parse waf configuration") + return errors.Wrap(err, "Cannot parse appsec configuration") } if wc.config.LogLevel == nil { @@ -124,8 +124,8 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { wc.config.Routines = 1 } - if wc.config.WaapConfig == "" && wc.config.WaapConfigPath == "" { - return fmt.Errorf("waap_config or waap_config_path must be set") + if wc.config.AppsecConfig == "" && wc.config.AppsecConfigPath == "" { + return fmt.Errorf("appsec_config or appsec_config_path must be set") } if wc.config.Name == "" { @@ -139,15 +139,15 @@ func (wc *WaapSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (w *WaapSource) GetMetrics() []prometheus.Collector { +func (w *AppsecSource) GetMetrics() []prometheus.Collector { return nil } -func (w *WaapSource) GetAggregMetrics() []prometheus.Collector { +func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { return nil } -func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { +func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { err := w.UnmarshalConfig(yamlConfig) if err != nil { return errors.Wrap(err, "unable to parse waf configuration") @@ -170,96 +170,95 @@ func (w *WaapSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.InChan = make(chan waf.ParsedRequest) - waapCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "waap_config")} + appsecCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "appsec_config")} - //let's load the associated waap_config: - if w.config.WaapConfigPath != "" { - err := waapCfg.LoadByPath(w.config.WaapConfigPath) + //let's load the associated appsec_config: + if w.config.AppsecConfigPath != "" { + err := appsecCfg.LoadByPath(w.config.AppsecConfigPath) if err != nil { - return fmt.Errorf("unable to load waap_config : %s", err) + return fmt.Errorf("unable to load appsec_config : %s", err) } - } else if w.config.WaapConfig != "" { - err := waapCfg.Load(w.config.WaapConfig) + } else if w.config.AppsecConfig != "" { + err := appsecCfg.Load(w.config.AppsecConfig) if err != nil { - return fmt.Errorf("unable to load waap_config : %s", err) + return fmt.Errorf("unable to load appsec_config : %s", err) } } else { - return fmt.Errorf("no waap_config provided") + return fmt.Errorf("no appsec_config provided") } - w.WaapRuntime, err = waapCfg.Build() + w.AppsecRuntime, err = appsecCfg.Build() if err != nil { - return fmt.Errorf("unable to build waap_config : %s", err) + return fmt.Errorf("unable to build appsec_config : %s", err) } - err = w.WaapRuntime.ProcessOnLoadRules() + err = w.AppsecRuntime.ProcessOnLoadRules() if err != nil { return fmt.Errorf("unable to process on load rules : %s", err) } - w.WaapRunners = make([]WaapRunner, w.config.Routines) + w.AppsecRunners = make([]AppsecRunner, w.config.Routines) for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { - - wafUUID := uuid.New().String() - //we copy WaapRutime for each runner - wrt := *w.WaapRuntime - runner := WaapRunner{ + appsecRunnerUUID := uuid.New().String() + //we copy AppsecRutime for each runner + wrt := *w.AppsecRuntime + runner := AppsecRunner{ inChan: w.InChan, - UUID: wafUUID, + UUID: appsecRunnerUUID, logger: w.logger.WithFields(log.Fields{ - "uuid": wafUUID, + "uuid": appsecRunnerUUID, }), WaapRuntime: &wrt, } - err := runner.Init(waapCfg.GetDataDir()) + err := runner.Init(appsecCfg.GetDataDir()) if err != nil { return fmt.Errorf("unable to initialize runner : %s", err) } - w.WaapRunners[nbRoutine] = runner + w.AppsecRunners[nbRoutine] = runner } - w.logger.Infof("Created %d waf runners", len(w.WaapRunners)) + w.logger.Infof("Created %d appsec runners", len(w.AppsecRunners)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec - w.mux.HandleFunc(w.config.Path, w.waapHandler) + w.mux.HandleFunc(w.config.Path, w.appsecHandler) return nil } -func (w *WaapSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { - return fmt.Errorf("WAF datasource does not support command line acquisition") +func (w *AppsecSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + return fmt.Errorf("AppSec datasource does not support command line acquisition") } -func (w *WaapSource) GetMode() string { +func (w *AppsecSource) GetMode() string { return w.config.Mode } -func (w *WaapSource) GetName() string { - return "waf" +func (w *AppsecSource) GetName() string { + return "appsec" } -func (w *WaapSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { - return fmt.Errorf("WAF datasource does not support command line acquisition") +func (w *AppsecSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("AppSec datasource does not support command line acquisition") } -func (w *WaapSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { +func (w *AppsecSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { w.outChan = out t.Go(func() error { - defer trace.CatchPanic("crowdsec/acquis/waf/live") + defer trace.CatchPanic("crowdsec/acquis/appsec/live") - w.logger.Infof("%d waf runner to start", len(w.WaapRunners)) - for _, runner := range w.WaapRunners { + w.logger.Infof("%d appsec runner to start", len(w.AppsecRunners)) + for _, runner := range w.AppsecRunners { runner := runner runner.outChan = out t.Go(func() error { - defer trace.CatchPanic("crowdsec/acquis/waf/live/runner") + defer trace.CatchPanic("crowdsec/acquis/appsec/live/runner") return runner.Run(t) }) } - w.logger.Infof("Starting WAF server on %s%s", w.config.ListenAddr, w.config.Path) + w.logger.Infof("Starting Appsec server on %s%s", w.config.ListenAddr, w.config.Path) t.Go(func() error { var err error if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { @@ -269,31 +268,31 @@ func (w *WaapSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er } if err != nil && err != http.ErrServerClosed { - return errors.Wrap(err, "WAF server failed") + return errors.Wrap(err, "Appsec server failed") } return nil }) <-t.Dying() - w.logger.Infof("Stopping WAF server on %s%s", w.config.ListenAddr, w.config.Path) + w.logger.Infof("Stopping Appsec server on %s%s", w.config.ListenAddr, w.config.Path) w.server.Shutdown(context.TODO()) return nil }) return nil } -func (w *WaapSource) CanRun() error { +func (w *AppsecSource) CanRun() error { return nil } -func (w *WaapSource) GetUuid() string { +func (w *AppsecSource) GetUuid() string { return w.config.UniqueId } -func (w *WaapSource) Dump() interface{} { +func (w *AppsecSource) Dump() interface{} { return w } -func (w *WaapSource) IsAuth(apiKey string) bool { +func (w *AppsecSource) IsAuth(apiKey string) bool { client := &http.Client{ Timeout: 200 * time.Millisecond, } @@ -317,7 +316,7 @@ func (w *WaapSource) IsAuth(apiKey string) bool { } // should this be in the runner ? -func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { +func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { apiKey := r.Header.Get(waf.APIKeyHeaderName) clientIP := r.Header.Get(waf.IPHeaderName) remoteIP := r.RemoteAddr @@ -348,19 +347,19 @@ func (w *WaapSource) waapHandler(rw http.ResponseWriter, r *http.Request) { } parsedRequest.WaapEngine = w.config.Name - WafReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() + AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.WaapEngine}).Inc() w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel if response.InBandInterrupt { - WafBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "waap_engine": parsedRequest.WaapEngine}).Inc() + AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.WaapEngine}).Inc() } - waapResponse := w.WaapRuntime.GenerateResponse(response) + appsecResponse := w.AppsecRuntime.GenerateResponse(response) - rw.WriteHeader(waapResponse.HTTPStatus) - body, err := json.Marshal(BodyResponse{Action: waapResponse.Action}) + rw.WriteHeader(appsecResponse.HTTPStatus) + body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action}) if err != nil { log.Errorf("unable to marshal response: %s", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/acquisition/modules/waap/waap_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go similarity index 89% rename from pkg/acquisition/modules/waap/waap_runner.go rename to pkg/acquisition/modules/appsec/appsec_runner.go index 5375a4d2c71..206e7523a55 100644 --- a/pkg/acquisition/modules/waap/waap_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -1,4 +1,4 @@ -package wafacquisition +package appsecacquisition import ( "fmt" @@ -16,7 +16,7 @@ import ( ) // that's the runtime structure of the WAAP as seen from the acquis -type WaapRunner struct { +type AppsecRunner struct { outChan chan types.Event inChan chan waf.ParsedRequest UUID string @@ -26,7 +26,7 @@ type WaapRunner struct { logger *log.Entry } -func (r *WaapRunner) Init(datadir string) error { +func (r *AppsecRunner) Init(datadir string) error { var err error fs := os.DirFS(datadir) @@ -101,7 +101,7 @@ func (r *WaapRunner) Init(datadir string) error { return nil } -func (r *WaapRunner) processRequest(tx waf.ExtendedTransaction, request *waf.ParsedRequest) error { +func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.ParsedRequest) error { var in *corazatypes.Interruption var err error request.Tx = tx @@ -185,21 +185,21 @@ func (r *WaapRunner) processRequest(tx waf.ExtendedTransaction, request *waf.Par return nil } -func (r *WaapRunner) ProcessInBandRules(request *waf.ParsedRequest) error { +func (r *AppsecRunner) ProcessInBandRules(request *waf.ParsedRequest) error { tx := waf.NewExtendedTransaction(r.WaapInbandEngine, request.UUID) r.WaapRuntime.InBandTx = tx err := r.processRequest(tx, request) return err } -func (r *WaapRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { +func (r *AppsecRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { tx := waf.NewExtendedTransaction(r.WaapOutbandEngine, request.UUID) r.WaapRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) return err } -func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleInBandInterrupt(request *waf.ParsedRequest) { //create the associated event for crowdsec itself evt, err := EventFromRequest(request) if err != nil { @@ -238,17 +238,17 @@ func (r *WaapRunner) handleInBandInterrupt(request *waf.ParsedRequest) { // Should the in band match trigger an overflow ? if r.WaapRuntime.Response.SendAlert { - waapOvlfw, err := WaapEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt) if err != nil { - r.logger.Errorf("unable to generate waap event : %s", err) + r.logger.Errorf("unable to generate appsec event : %s", err) return } - r.outChan <- *waapOvlfw + r.outChan <- *appsecOvlfw } } } -func (r *WaapRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { evt, err := EventFromRequest(request) if err != nil { //let's not interrupt the pipeline for this @@ -274,24 +274,24 @@ func (r *WaapRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { // Should the match trigger an overflow ? if r.WaapRuntime.Response.SendAlert { - waapOvlfw, err := WaapEventGeneration(evt) + appsecOvlfw, err := AppsecEventGeneration(evt) if err != nil { r.logger.Errorf("unable to generate waap event : %s", err) return } - r.outChan <- *waapOvlfw + r.outChan <- *appsecOvlfw } } } -func (r *WaapRunner) handleRequest(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { r.logger.Debugf("Requests handled by runner %s", request.UUID) r.WaapRuntime.ClearResponse() request.IsInBand = true request.IsOutBand = false - //to measure the time spent in the WAF + //to measure the time spent in the Application Security Engine startParsing := time.Now() //inband WAAP rules @@ -306,7 +306,7 @@ func (r *WaapRunner) handleRequest(request *waf.ParsedRequest) { } elapsed := time.Since(startParsing) - WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) + AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- r.WaapRuntime.Response @@ -329,7 +329,7 @@ func (r *WaapRunner) handleRequest(request *waf.ParsedRequest) { } } -func (r *WaapRunner) Run(t *tomb.Tomb) error { +func (r *AppsecRunner) Run(t *tomb.Tomb) error { r.logger.Infof("Waap Runner ready to process event") for { select { diff --git a/pkg/acquisition/modules/appsec/metrics.go b/pkg/acquisition/modules/appsec/metrics.go new file mode 100644 index 00000000000..9aa3c8bde7a --- /dev/null +++ b/pkg/acquisition/modules/appsec/metrics.go @@ -0,0 +1,54 @@ +package appsecacquisition + +import "github.com/prometheus/client_golang/prometheus" + +var AppsecGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: "cs_appsec_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband Application Security Engine.", + Name: "cs_appsec_inband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: "cs_appsec_outband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_reqs_total", + Help: "Total events processed by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +var AppsecBlockCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_block_total", + Help: "Total events blocked by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +var AppsecRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_rule_hits", + Help: "Count of triggered rule, by rule_name, type (inband/outofband), appsec_engine and source", + }, + []string{"rule_name", "type", "appsec_engine", "source"}, +) diff --git a/pkg/acquisition/modules/waap/rx_operator.go b/pkg/acquisition/modules/appsec/rx_operator.go similarity index 98% rename from pkg/acquisition/modules/waap/rx_operator.go rename to pkg/acquisition/modules/appsec/rx_operator.go index f1335c7e3aa..43aaf9e94be 100644 --- a/pkg/acquisition/modules/waap/rx_operator.go +++ b/pkg/acquisition/modules/appsec/rx_operator.go @@ -1,4 +1,4 @@ -package wafacquisition +package appsecacquisition import ( "fmt" diff --git a/pkg/acquisition/modules/waap/utils.go b/pkg/acquisition/modules/appsec/utils.go similarity index 89% rename from pkg/acquisition/modules/waap/utils.go rename to pkg/acquisition/modules/appsec/utils.go index cf02abae719..ebec71ef4ad 100644 --- a/pkg/acquisition/modules/waap/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -1,4 +1,4 @@ -package wafacquisition +package appsecacquisition import ( "encoding/json" @@ -15,13 +15,13 @@ import ( log "github.com/sirupsen/logrus" ) -func WaapEventGeneration(inEvt types.Event) (*types.Event, error) { +func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { //if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI if !inEvt.Waap.HasInBandMatches { return nil, nil } evt := types.Event{} - evt.Type = types.WAAP + evt.Type = types.APPSEC evt.Process = true source := models.Source{ Value: ptr.Of(inEvt.Parsed["source_ip"]), @@ -53,7 +53,7 @@ func WaapEventGeneration(inEvt types.Event) (*types.Event, error) { alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"waf"} //don't know what to do about this alert.Leakspeed = ptr.Of("") - msg := fmt.Sprintf("WAF alert: %s", inEvt.Waap.MatchedRules.GetName()) + msg := fmt.Sprintf("Application Security Engine alert: %s", inEvt.Waap.MatchedRules.GetName()) alert.Message = &msg alert.Scenario = ptr.Of(inEvt.Waap.MatchedRules.GetName()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() alert.ScenarioHash = ptr.Of(inEvt.Waap.MatchedRules.GetHash()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() @@ -91,18 +91,18 @@ func EventFromRequest(r *waf.ParsedRequest) (types.Event, error) { evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "crowdsec-waap"}, + Labels: map[string]string{"type": "crowdsec-waap"}, //FIXME: use the labels from the acquis Process: true, - Module: "waap", - Src: "waap", + Module: "appsec", + Src: "appsec", Raw: "dummy-waap-data", //we discard empty Line.Raw items :) } - evt.Waap = types.WaapEvent{} + evt.Waap = types.AppsecEvent{} return evt, nil } -func LogWaapEvent(evt *types.Event, logger *log.Entry) { +func LogAppsecEvent(evt *types.Event, logger *log.Entry) { req := evt.Parsed["target_uri"] if len(req) > 12 { req = req[:10] + ".." @@ -125,12 +125,12 @@ func LogWaapEvent(evt *types.Event, logger *log.Entry) { "module": "waf", "source": evt.Parsed["source_ip"], "target_uri": req, - }).Debugf("%s triggerd non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + }).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) } } -func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequest) error { +func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequest) error { if evt == nil { //an error was already emitted, let's not spam the logs @@ -215,7 +215,7 @@ func (r *WaapRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedReques r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) } - WafRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "waap_engine": req.WaapEngine}).Inc() + AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.WaapEngine}).Inc() corazaRule := map[string]interface{}{ "id": rule.Rule().ID(), diff --git a/pkg/acquisition/modules/waap/metrics.go b/pkg/acquisition/modules/waap/metrics.go deleted file mode 100644 index fd917a8b2d3..00000000000 --- a/pkg/acquisition/modules/waap/metrics.go +++ /dev/null @@ -1,54 +0,0 @@ -package wafacquisition - -import "github.com/prometheus/client_golang/prometheus" - -var WafGlobalParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the WAF.", - Name: "cs_waf_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafInbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the inband WAF.", - Name: "cs_waf_inband_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafOutbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the WAF.", - Name: "cs_waf_outband_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"source"}, -) - -var WafReqCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_waf_reqs_total", - Help: "Total events processed by the WAF.", - }, - []string{"source", "waap_engine"}, -) - -var WafBlockCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_waf_block_total", - Help: "Total events blocked by the WAF.", - }, - []string{"source", "waap_engine"}, -) - -var WafRuleHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_waf_rule_hits", - Help: "Count of triggered rule, by rule_name, type (inband/outofband), waap_engine and source", - }, - []string{"rule_name", "type", "waap_engine", "source"}, -) diff --git a/pkg/types/waap_event.go b/pkg/types/appsec_event.go similarity index 98% rename from pkg/types/waap_event.go rename to pkg/types/appsec_event.go index 9b64d8d7a34..83ee06fd7e7 100644 --- a/pkg/types/waap_event.go +++ b/pkg/types/appsec_event.go @@ -19,7 +19,7 @@ len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 type MatchedRules []map[string]interface{} -type WaapEvent struct { +type AppsecEvent struct { HasInBandMatches, HasOutBandMatches bool MatchedRules Vars map[string]string @@ -44,7 +44,7 @@ const ( Kind Field = "kind" ) -func (w WaapEvent) GetVar(varName string) string { +func (w AppsecEvent) GetVar(varName string) string { if w.Vars == nil { return "" } diff --git a/pkg/types/event.go b/pkg/types/event.go index 0e8de7f8e29..2456df8672f 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -13,7 +13,7 @@ import ( const ( LOG = iota OVFLW - WAAP + APPSEC ) // Event is the structure representing a runtime event (log or overflow) @@ -41,7 +41,7 @@ type Event struct { StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line - Waap WaapEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"` + Waap AppsecEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"` /* Meta is the only part that will make it to the API - it should be normalized */ Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } From bff93d7b014e48c6da3538d529f504875c5cb16e Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 21:58:29 +0100 Subject: [PATCH 235/263] appsec renaming, part 2 --- cmd/crowdsec-cli/waap_rules.go | 2 +- cmd/crowdsec/crowdsec.go | 4 +- .../modules/appsec/appsec_runner.go | 2 +- pkg/acquisition/modules/appsec/utils.go | 2 +- pkg/waf/coraza_logger.go | 2 +- pkg/waf/loader.go | 30 ++-- pkg/waf/request.go | 40 ++--- pkg/waf/waap.go | 152 +++++++++--------- pkg/waf/waap_rules_collection.go | 74 ++++----- pkg/waf/waf_helpers.go | 8 +- 10 files changed, 149 insertions(+), 167 deletions(-) diff --git a/cmd/crowdsec-cli/waap_rules.go b/cmd/crowdsec-cli/waap_rules.go index 342ff63eed2..ade49aaacd7 100644 --- a/cmd/crowdsec-cli/waap_rules.go +++ b/cmd/crowdsec-cli/waap_rules.go @@ -127,7 +127,7 @@ func WaapRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args hub, _ := require.Hub(csConfig, nil) for _, name := range args { hubItem := hub.GetItem(itemType.name, name) - waapRule := waf.WaapCollectionConfig{} + waapRule := waf.AppsecCollectionConfig{} yamlContent, err := os.ReadFile(hubItem.State.LocalPath) if err != nil { return fmt.Errorf("unable to read file %s : %s", hubItem.State.LocalPath, err) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 686d32f2038..64c4797279b 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -34,8 +34,8 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er return nil, fmt.Errorf("while loading scenarios: %w", err) } - if err := waf.LoadWaapRules(hub); err != nil { - return nil, fmt.Errorf("while loading waap rules: %w", err) + if err := waf.LoadAppsecRules(hub); err != nil { + return nil, fmt.Errorf("while loading appsec rules: %w", err) } if err := LoadAcquisition(cConfig); err != nil { diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 206e7523a55..0aeca248990 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -20,7 +20,7 @@ type AppsecRunner struct { outChan chan types.Event inChan chan waf.ParsedRequest UUID string - WaapRuntime *waf.WaapRuntimeConfig //this holds the actual waap runtime config, rules, remediations, hooks etc. + WaapRuntime *waf.AppsecRuntimeConfig //this holds the actual waap runtime config, rules, remediations, hooks etc. WaapInbandEngine coraza.WAF WaapOutbandEngine coraza.WAF logger *log.Entry diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index ebec71ef4ad..ea6c1670e29 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -206,7 +206,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequ hash := "NOT_SET" ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID()) - if details, ok := waf.WaapRulesDetails[rule.Rule().ID()]; ok { + if details, ok := waf.AppsecRulesDetails[rule.Rule().ID()]; ok { //Only set them for custom rules, not for rules written in seclang name = details.Name version = details.Version diff --git a/pkg/waf/coraza_logger.go b/pkg/waf/coraza_logger.go index d7e680c76d7..dfeeb669e47 100644 --- a/pkg/waf/coraza_logger.go +++ b/pkg/waf/coraza_logger.go @@ -36,7 +36,7 @@ func (e *crzLogEvent) Msg(msg string) { } /*this is a hack. As we want to have per-level rule debug but it's not allowed by coraza/modsec, if a rule ID is flagged to be in debug mode, the - .Int("rule_id", ) call will set the log_level of the event to debug. However, given the logger is global to the waap-runner, + .Int("rule_id", ) call will set the log_level of the event to debug. However, given the logger is global to the appsec-runner, we are switching forth and back the log level of the logger*/ oldLvl := e.logger.Logger.GetLevel() diff --git a/pkg/waf/loader.go b/pkg/waf/loader.go index 112cf277407..f80f09f1266 100644 --- a/pkg/waf/loader.go +++ b/pkg/waf/loader.go @@ -8,51 +8,51 @@ import ( "gopkg.in/yaml.v2" ) -var waapRules map[string]WaapCollectionConfig = make(map[string]WaapCollectionConfig) //FIXME: would probably be better to have a struct for this +var appsecRules map[string]AppsecCollectionConfig = make(map[string]AppsecCollectionConfig) //FIXME: would probably be better to have a struct for this var hub *cwhub.Hub //FIXME: this is a temporary hack to make the hub available in the package -func LoadWaapRules(hubInstance *cwhub.Hub) error { +func LoadAppsecRules(hubInstance *cwhub.Hub) error { hub = hubInstance - for _, hubWafRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { + for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { //log.Infof("loading %s", hubWafRuleItem.LocalPath) - if !hubWafRuleItem.State.Installed { + if !hubAppsecRuleItem.State.Installed { continue } - content, err := os.ReadFile(hubWafRuleItem.State.LocalPath) + content, err := os.ReadFile(hubAppsecRuleItem.State.LocalPath) if err != nil { - log.Warnf("unable to read file %s : %s", hubWafRuleItem.State.LocalPath, err) + log.Warnf("unable to read file %s : %s", hubAppsecRuleItem.State.LocalPath, err) continue } - var rule WaapCollectionConfig + var rule AppsecCollectionConfig err = yaml.UnmarshalStrict(content, &rule) if err != nil { - log.Warnf("unable to unmarshal file %s : %s", hubWafRuleItem.State.LocalPath, err) + log.Warnf("unable to unmarshal file %s : %s", hubAppsecRuleItem.State.LocalPath, err) continue } if rule.Type != WAAP_RULE { - log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubWafRuleItem.State.LocalPath) + log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubAppsecRuleItem.State.LocalPath) continue } - rule.hash = hubWafRuleItem.State.LocalHash - rule.version = hubWafRuleItem.Version + rule.hash = hubAppsecRuleItem.State.LocalHash + rule.version = hubAppsecRuleItem.Version - log.Infof("Adding %s to waap rules", rule.Name) + log.Infof("Adding %s to appsec rules", rule.Name) - waapRules[rule.Name] = rule + appsecRules[rule.Name] = rule } - if len(waapRules) == 0 { - log.Debugf("No waap rules found") + if len(appsecRules) == 0 { + log.Debugf("No appsec rules found") } return nil } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 5e792a13bbf..66b72467cd1 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -23,25 +23,25 @@ const ( ) type ParsedRequest struct { - RemoteAddr string `json:"remote_addr,omitempty"` - Host string `json:"host,omitempty"` - ClientIP string `json:"client_ip,omitempty"` - URI string `json:"uri,omitempty"` - Args url.Values `json:"args,omitempty"` - ClientHost string `json:"client_host,omitempty"` - Headers http.Header `json:"headers,omitempty"` - URL *url.URL `json:"url,omitempty"` - Method string `json:"method,omitempty"` - Proto string `json:"proto,omitempty"` - Body []byte `json:"body,omitempty"` - TransferEncoding []string `json:"transfer_encoding,omitempty"` - UUID string `json:"uuid,omitempty"` - Tx ExtendedTransaction `json:"transaction,omitempty"` - ResponseChannel chan WaapTempResponse `json:"-"` - IsInBand bool `json:"-"` - IsOutBand bool `json:"-"` - WaapEngine string `json:"waap_engine,omitempty"` - RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` + RemoteAddr string `json:"remote_addr,omitempty"` + Host string `json:"host,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + URI string `json:"uri,omitempty"` + Args url.Values `json:"args,omitempty"` + ClientHost string `json:"client_host,omitempty"` + Headers http.Header `json:"headers,omitempty"` + URL *url.URL `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Proto string `json:"proto,omitempty"` + Body []byte `json:"body,omitempty"` + TransferEncoding []string `json:"transfer_encoding,omitempty"` + UUID string `json:"uuid,omitempty"` + Tx ExtendedTransaction `json:"transaction,omitempty"` + ResponseChannel chan AppsecTempResponse `json:"-"` + IsInBand bool `json:"-"` + IsOutBand bool `json:"-"` + WaapEngine string `json:"waap_engine,omitempty"` + RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` } type ReqDumpFilter struct { @@ -339,7 +339,7 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { Body: body, Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args TransferEncoding: r.TransferEncoding, - ResponseChannel: make(chan WaapTempResponse), + ResponseChannel: make(chan AppsecTempResponse), RemoteAddrNormalized: remoteAddrNormalized, }, nil } diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 228f904c91e..34ebc079fcb 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -36,13 +36,13 @@ func (h *Hook) Build(hookStage int) error { ctx := map[string]interface{}{} switch hookStage { case hookOnLoad: - ctx = GetOnLoadEnv(&WaapRuntimeConfig{}) + ctx = GetOnLoadEnv(&AppsecRuntimeConfig{}) case hookPreEval: - ctx = GetPreEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetPreEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}) case hookPostEval: - ctx = GetPostEvalEnv(&WaapRuntimeConfig{}, &ParsedRequest{}) + ctx = GetPostEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}) case hookOnMatch: - ctx = GetOnMatchEnv(&WaapRuntimeConfig{}, &ParsedRequest{}, types.Event{}) + ctx = GetOnMatchEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}, types.Event{}) } opts := exprhelpers.GetExprOptions(ctx) if h.Filter != "" { @@ -62,7 +62,7 @@ func (h *Hook) Build(hookStage int) error { return nil } -type WaapTempResponse struct { +type AppsecTempResponse struct { InBandInterrupt bool OutOfBandInterrupt bool Action string //allow, deny, captcha, log @@ -71,17 +71,17 @@ type WaapTempResponse struct { SendAlert bool //do we send an alert on rule match } -type WaapSubEngineOpts struct { +type AppsecSubEngineOpts struct { DisableBodyInspection bool `yaml:"disable_body_inspection"` RequestBodyInMemoryLimit *int `yaml:"request_body_in_memory_limit"` } -// runtime version of WaapConfig -type WaapRuntimeConfig struct { +// runtime version of AppsecConfig +type AppsecRuntimeConfig struct { Name string - OutOfBandRules []WaapCollection + OutOfBandRules []AppsecCollection - InBandRules []WaapCollection + InBandRules []AppsecCollection DefaultRemediation string RemediationByTag map[string]string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME @@ -91,13 +91,13 @@ type WaapRuntimeConfig struct { CompiledPostEval []Hook CompiledOnMatch []Hook CompiledVariablesTracking []*regexp.Regexp - Config *WaapConfig + Config *AppsecConfig //CorazaLogger debuglog.Logger //those are ephemeral, created/destroyed with every req OutOfBandTx ExtendedTransaction //is it a good idea ? InBandTx ExtendedTransaction //is it a good idea ? - Response WaapTempResponse + Response AppsecTempResponse //should we store matched rules here ? Logger *log.Entry @@ -110,29 +110,29 @@ type WaapRuntimeConfig struct { DisabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME } -type WaapConfig struct { - Name string `yaml:"name"` - OutOfBandRules []string `yaml:"outofband_rules"` - InBandRules []string `yaml:"inband_rules"` - DefaultRemediation string `yaml:"default_remediation"` - DefaultPassAction string `yaml:"default_pass_action"` - BlockedHTTPCode int `yaml:"blocked_http_code"` - PassedHTTPCode int `yaml:"passed_http_code"` - OnLoad []Hook `yaml:"on_load"` - PreEval []Hook `yaml:"pre_eval"` - PostEval []Hook `yaml:"post_eval"` - OnMatch []Hook `yaml:"on_match"` - VariablesTracking []string `yaml:"variables_tracking"` - InbandOptions WaapSubEngineOpts `yaml:"inband_options"` - OutOfBandOptions WaapSubEngineOpts `yaml:"outofband_options"` +type AppsecConfig struct { + Name string `yaml:"name"` + OutOfBandRules []string `yaml:"outofband_rules"` + InBandRules []string `yaml:"inband_rules"` + DefaultRemediation string `yaml:"default_remediation"` + DefaultPassAction string `yaml:"default_pass_action"` + BlockedHTTPCode int `yaml:"blocked_http_code"` + PassedHTTPCode int `yaml:"passed_http_code"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + PostEval []Hook `yaml:"post_eval"` + OnMatch []Hook `yaml:"on_match"` + VariablesTracking []string `yaml:"variables_tracking"` + InbandOptions AppsecSubEngineOpts `yaml:"inband_options"` + OutOfBandOptions AppsecSubEngineOpts `yaml:"outofband_options"` LogLevel *log.Level `yaml:"log_level"` Logger *log.Entry `yaml:"-"` } -func (w *WaapRuntimeConfig) ClearResponse() { +func (w *AppsecRuntimeConfig) ClearResponse() { log.Debugf("#-> %p", w) - w.Response = WaapTempResponse{} + w.Response = AppsecTempResponse{} log.Debugf("-> %p", w.Config) w.Response.Action = w.Config.DefaultPassAction w.Response.HTTPResponseCode = w.Config.PassedHTTPCode @@ -140,7 +140,7 @@ func (w *WaapRuntimeConfig) ClearResponse() { w.Response.SendAlert = true } -func (wc *WaapConfig) LoadByPath(file string) error { +func (wc *AppsecConfig) LoadByPath(file string) error { wc.Logger.Debugf("loading config %s", file) @@ -183,33 +183,33 @@ func (wc *WaapConfig) LoadByPath(file string) error { return nil } -func (wc *WaapConfig) Load(configName string) error { - waapConfigs := hub.GetItemMap(cwhub.WAAP_CONFIGS) +func (wc *AppsecConfig) Load(configName string) error { + appsecConfigs := hub.GetItemMap(cwhub.WAAP_CONFIGS) - for _, hubWaapConfigItem := range waapConfigs { - if !hubWaapConfigItem.State.Installed { + for _, hubAppsecConfigItem := range appsecConfigs { + if !hubAppsecConfigItem.State.Installed { continue } - if hubWaapConfigItem.Name != configName { + if hubAppsecConfigItem.Name != configName { continue } - wc.Logger.Infof("loading %s", hubWaapConfigItem.State.LocalPath) - err := wc.LoadByPath(hubWaapConfigItem.State.LocalPath) + wc.Logger.Infof("loading %s", hubAppsecConfigItem.State.LocalPath) + err := wc.LoadByPath(hubAppsecConfigItem.State.LocalPath) if err != nil { - return fmt.Errorf("unable to load waap-config %s : %s", hubWaapConfigItem.State.LocalPath, err) + return fmt.Errorf("unable to load appsec-config %s : %s", hubAppsecConfigItem.State.LocalPath, err) } return nil } - return fmt.Errorf("no waap-config found for %s", configName) + return fmt.Errorf("no appsec-config found for %s", configName) } -func (wc *WaapConfig) GetDataDir() string { +func (wc *AppsecConfig) GetDataDir() string { return hub.GetDataDir() } -func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { - ret := &WaapRuntimeConfig{Logger: wc.Logger.WithField("component", "waap_runtime_config")} +func (wc *AppsecConfig) Build() (*AppsecRuntimeConfig, error) { + ret := &AppsecRuntimeConfig{Logger: wc.Logger.WithField("component", "appsec_runtime_config")} ret.Name = wc.Name ret.Config = wc ret.DefaultRemediation = wc.DefaultRemediation @@ -280,12 +280,12 @@ func (wc *WaapConfig) Build() (*WaapRuntimeConfig, error) { return ret, nil } -func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { +func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error { for _, rule := range w.CompiledOnLoad { if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - return fmt.Errorf("unable to run waap on_load filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run appsec on_load filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -301,7 +301,7 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { for _, applyExpr := range rule.ApplyExpr { _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply waap on_load expr: %s", err) + log.Errorf("unable to apply appsec on_load expr: %s", err) continue } } @@ -309,13 +309,13 @@ func (w *WaapRuntimeConfig) ProcessOnLoadRules() error { return nil } -func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { +func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { for _, rule := range w.CompiledOnMatch { if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - return fmt.Errorf("unable to run waap on_match filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run appsec on_match filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -331,7 +331,7 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type for _, applyExpr := range rule.ApplyExpr { _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply waap on_match expr: %s", err) + log.Errorf("unable to apply appsec on_match expr: %s", err) continue } } @@ -339,12 +339,12 @@ func (w *WaapRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt type return nil } -func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { +func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - return fmt.Errorf("unable to run waap pre_eval filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run appsec pre_eval filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -361,7 +361,7 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { for _, applyExpr := range rule.ApplyExpr { _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply waap pre_eval expr: %s", err) + log.Errorf("unable to apply appsec pre_eval expr: %s", err) continue } } @@ -370,12 +370,12 @@ func (w *WaapRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { return nil } -func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { +func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { for _, rule := range w.CompiledPostEval { if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - return fmt.Errorf("unable to run waap post_eval filter %s : %w", rule.Filter, err) + return fmt.Errorf("unable to run appsec post_eval filter %s : %w", rule.Filter, err) } switch t := output.(type) { case bool: @@ -392,7 +392,7 @@ func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { for _, applyExpr := range rule.ApplyExpr { _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply waap post_eval expr: %s", err) + log.Errorf("unable to apply appsec post_eval expr: %s", err) continue } } @@ -401,101 +401,101 @@ func (w *WaapRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { return nil } -func (w *WaapRuntimeConfig) RemoveInbandRuleByID(id int) error { +func (w *AppsecRuntimeConfig) RemoveInbandRuleByID(id int) error { w.Logger.Debugf("removing inband rule %d", id) return w.InBandTx.RemoveRuleByIDWithError(id) } -func (w *WaapRuntimeConfig) RemoveOutbandRuleByID(id int) error { +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByID(id int) error { w.Logger.Debugf("removing outband rule %d", id) return w.OutOfBandTx.RemoveRuleByIDWithError(id) } -func (w *WaapRuntimeConfig) RemoveInbandRuleByTag(tag string) error { +func (w *AppsecRuntimeConfig) RemoveInbandRuleByTag(tag string) error { w.Logger.Debugf("removing inband rule with tag %s", tag) return w.InBandTx.RemoveRuleByTagWithError(tag) } -func (w *WaapRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { w.Logger.Debugf("removing outband rule with tag %s", tag) return w.OutOfBandTx.RemoveRuleByTagWithError(tag) } -func (w *WaapRuntimeConfig) RemoveInbandRuleByName(name string) error { +func (w *AppsecRuntimeConfig) RemoveInbandRuleByName(name string) error { tag := fmt.Sprintf("crowdsec-%s", name) w.Logger.Debugf("removing inband rule %s", tag) return w.InBandTx.RemoveRuleByTagWithError(tag) } -func (w *WaapRuntimeConfig) RemoveOutbandRuleByName(name string) error { +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByName(name string) error { tag := fmt.Sprintf("crowdsec-%s", name) w.Logger.Debugf("removing outband rule %s", tag) return w.OutOfBandTx.RemoveRuleByTagWithError(tag) } -func (w *WaapRuntimeConfig) CancelEvent() error { +func (w *AppsecRuntimeConfig) CancelEvent() error { w.Logger.Debugf("canceling event") w.Response.SendEvent = false return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByID(id int) error { +func (w *AppsecRuntimeConfig) DisableInBandRuleByID(id int) error { w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, id) return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByName(name string) error { +func (w *AppsecRuntimeConfig) DisableInBandRuleByName(name string) error { tagValue := fmt.Sprintf("crowdsec-%s", name) w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tagValue) return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableInBandRuleByTag(tag string) error { +func (w *AppsecRuntimeConfig) DisableInBandRuleByTag(tag string) error { w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tag) return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByID(id int) error { +func (w *AppsecRuntimeConfig) DisableOutBandRuleByID(id int) error { w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, id) return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByName(name string) error { +func (w *AppsecRuntimeConfig) DisableOutBandRuleByName(name string) error { tagValue := fmt.Sprintf("crowdsec-%s", name) w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tagValue) return nil } // Disable a rule at load time, meaning it will not run for any request -func (w *WaapRuntimeConfig) DisableOutBandRuleByTag(tag string) error { +func (w *AppsecRuntimeConfig) DisableOutBandRuleByTag(tag string) error { w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tag) return nil } -func (w *WaapRuntimeConfig) SendEvent() error { +func (w *AppsecRuntimeConfig) SendEvent() error { w.Logger.Debugf("sending event") w.Response.SendEvent = true return nil } -func (w *WaapRuntimeConfig) SendAlert() error { +func (w *AppsecRuntimeConfig) SendAlert() error { w.Logger.Debugf("sending alert") w.Response.SendAlert = true return nil } -func (w *WaapRuntimeConfig) CancelAlert() error { +func (w *AppsecRuntimeConfig) CancelAlert() error { w.Logger.Debugf("canceling alert") w.Response.SendAlert = false return nil } -func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { +func (w *AppsecRuntimeConfig) SetActionByTag(tag string, action string) error { if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } @@ -504,7 +504,7 @@ func (w *WaapRuntimeConfig) SetActionByTag(tag string, action string) error { return nil } -func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { +func (w *AppsecRuntimeConfig) SetActionByID(id int, action string) error { if w.RemediationById == nil { w.RemediationById = make(map[int]string) } @@ -513,7 +513,7 @@ func (w *WaapRuntimeConfig) SetActionByID(id int, action string) error { return nil } -func (w *WaapRuntimeConfig) SetActionByName(name string, action string) error { +func (w *AppsecRuntimeConfig) SetActionByName(name string, action string) error { if w.RemediationByTag == nil { w.RemediationByTag = make(map[string]string) } @@ -523,7 +523,7 @@ func (w *WaapRuntimeConfig) SetActionByName(name string, action string) error { return nil } -func (w *WaapRuntimeConfig) SetAction(action string) error { +func (w *AppsecRuntimeConfig) SetAction(action string) error { //log.Infof("setting to %s", action) w.Logger.Debugf("setting action to %s", action) switch action { @@ -544,7 +544,7 @@ func (w *WaapRuntimeConfig) SetAction(action string) error { return nil } -func (w *WaapRuntimeConfig) SetHTTPCode(code int) error { +func (w *AppsecRuntimeConfig) SetHTTPCode(code int) error { w.Logger.Debugf("setting http code to %d", code) w.Response.HTTPResponseCode = code return nil @@ -555,7 +555,7 @@ type BodyResponse struct { HTTPStatus int `json:"http_status"` } -func (w *WaapRuntimeConfig) GenerateResponse(response WaapTempResponse) BodyResponse { +func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse) BodyResponse { resp := BodyResponse{} //if there is no interrupt, we should allow with default code if !response.InBandInterrupt { diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index c699a724f0b..fdb96ba75cb 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -5,15 +5,13 @@ import ( "path/filepath" "strings" - corazatypes "github.com/crowdsecurity/coraza/v3/types" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" log "github.com/sirupsen/logrus" ) -// to be filled w/ seb update -type WaapCollection struct { +type AppsecCollection struct { collectionName string Rules []string } @@ -21,7 +19,7 @@ type WaapCollection struct { var WAAP_RULE = "waap-rule" // to be filled w/ seb update -type WaapCollectionConfig struct { +type AppsecCollectionConfig struct { Type string `yaml:"type"` Name string `yaml:"name"` Debug bool `yaml:"debug"` @@ -46,25 +44,25 @@ type RulesDetails struct { // Should it be a global ? // Is using the id is a good idea ? might be too specific to coraza and not easily reusable -var WaapRulesDetails = make(map[int]RulesDetails) +var AppsecRulesDetails = make(map[int]RulesDetails) -func LoadCollection(pattern string) ([]WaapCollection, error) { +func LoadCollection(pattern string) ([]AppsecCollection, error) { //FIXME: have a proper logger here, inheriting from waap-config to have consistent log levels - ret := make([]WaapCollection, 0) + ret := make([]AppsecCollection, 0) - for _, waapRule := range waapRules { + for _, appsecRule := range appsecRules { - tmpMatch, err := exprhelpers.Match(pattern, waapRule.Name) + tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name) if err != nil { - log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) + log.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) continue } matched, ok := tmpMatch.(bool) if !ok { - log.Errorf("unable to match %s with %s : %s", waapRule.Name, pattern, err) + log.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) continue } @@ -72,12 +70,12 @@ func LoadCollection(pattern string) ([]WaapCollection, error) { continue } - waapCol := WaapCollection{ - collectionName: waapRule.Name, + appsecCol := AppsecCollection{ + collectionName: appsecRule.Name, } - if waapRule.SecLangFilesRules != nil { - for _, rulesFile := range waapRule.SecLangFilesRules { + if appsecRule.SecLangFilesRules != nil { + for _, rulesFile := range appsecRule.SecLangFilesRules { fullPath := filepath.Join(hub.GetDataDir(), rulesFile) c, err := os.ReadFile(fullPath) if err != nil { @@ -91,64 +89,48 @@ func LoadCollection(pattern string) ([]WaapCollection, error) { if strings.TrimSpace(line) == "" { continue } - waapCol.Rules = append(waapCol.Rules, line) + appsecCol.Rules = append(appsecCol.Rules, line) } } } - if waapRule.SecLangRules != nil { - waapCol.Rules = append(waapCol.Rules, waapRule.SecLangRules...) + if appsecRule.SecLangRules != nil { + appsecCol.Rules = append(appsecCol.Rules, appsecRule.SecLangRules...) } - if waapRule.Rules != nil { - for _, rule := range waapRule.Rules { - strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, waapRule.Name) + if appsecRule.Rules != nil { + for _, rule := range appsecRule.Rules { + strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, appsecRule.Name) if err != nil { log.Errorf("unable to convert rule %s : %s", rule.Name, err) return nil, err } log.Debugf("Adding rule %s", strRule) - waapCol.Rules = append(waapCol.Rules, strRule) + appsecCol.Rules = append(appsecCol.Rules, strRule) //We only take the first id, as it's the one of the "main" rule - if _, ok := WaapRulesDetails[int(rulesId[0])]; !ok { - WaapRulesDetails[int(rulesId[0])] = RulesDetails{ + if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok { + AppsecRulesDetails[int(rulesId[0])] = RulesDetails{ LogLevel: log.InfoLevel, - Hash: waapRule.hash, - Version: waapRule.version, - Name: waapRule.Name, + Hash: appsecRule.hash, + Version: appsecRule.version, + Name: appsecRule.Name, } } else { log.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) } for _, id := range rulesId { - SetRuleDebug(int(id), waapRule.Debug) + SetRuleDebug(int(id), appsecRule.Debug) } } } - ret = append(ret, waapCol) + ret = append(ret, appsecCol) } return ret, nil } -func (wcc WaapCollectionConfig) LoadCollection(collection string) (WaapCollection, error) { - return WaapCollection{}, nil -} - -func (w WaapCollection) Check() error { - return nil -} - -func (w WaapCollection) Eval(req ParsedRequest) (*corazatypes.Interruption, error) { - return nil, nil -} - -func (w WaapCollection) GetDisplayName() string { - return w.collectionName -} - -func (w WaapCollection) String() string { +func (w AppsecCollection) String() string { ret := "" for _, rule := range w.Rules { ret += rule + "\n" diff --git a/pkg/waf/waf_helpers.go b/pkg/waf/waf_helpers.go index a978648778e..fa43c88482a 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/waf/waf_helpers.go @@ -4,7 +4,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { +func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} { return map[string]interface{}{ "RemoveInBandRuleByID": w.DisableInBandRuleByID, "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, @@ -18,7 +18,7 @@ func GetOnLoadEnv(w *WaapRuntimeConfig) map[string]interface{} { } } -func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { +func GetPreEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} { return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, @@ -34,7 +34,7 @@ func GetPreEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]inte } } -func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]interface{} { +func GetPostEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} { return map[string]interface{}{ "IsInBand": request.IsInBand, "IsOutBand": request.IsOutBand, @@ -42,7 +42,7 @@ func GetPostEvalEnv(w *WaapRuntimeConfig, request *ParsedRequest) map[string]int } } -func GetOnMatchEnv(w *WaapRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { +func GetOnMatchEnv(w *AppsecRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { return map[string]interface{}{ "evt": evt, "req": request, From 80466902195c89f8415df99d55a1e4e25869cc2c Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 22:07:34 +0100 Subject: [PATCH 236/263] appsec renaming, part 3 --- .../{waap_rules.go => appsec_rules.go} | 135 +++++++++--------- cmd/crowdsec-cli/itemcommands.go | 64 ++++----- cmd/crowdsec-cli/main.go | 4 +- pkg/acquisition/modules/appsec/appsec.go | 6 +- 4 files changed, 104 insertions(+), 105 deletions(-) rename cmd/crowdsec-cli/{waap_rules.go => appsec_rules.go} (53%) diff --git a/cmd/crowdsec-cli/waap_rules.go b/cmd/crowdsec-cli/appsec_rules.go similarity index 53% rename from cmd/crowdsec-cli/waap_rules.go rename to cmd/crowdsec-cli/appsec_rules.go index ade49aaacd7..08aa34902b2 100644 --- a/cmd/crowdsec-cli/waap_rules.go +++ b/cmd/crowdsec-cli/appsec_rules.go @@ -16,18 +16,18 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" ) -func NewWaapRulesCmd() *cobra.Command { - cmdWaapRules := &cobra.Command{ - Use: "waap-rules [waap-rule]...", - Short: "Manage hub waap rules", - Example: `cscli waap-rules list -a -cscli waap-rules install crowdsecurity/crs -cscli waap-rules inspect crowdsecurity/crs -cscli waap-rules upgrade crowdsecurity/crs -cscli waap-rules remove crowdsecurity/crs +func NewAppsecRulesCmd() *cobra.Command { + cmdAppsecRules := &cobra.Command{ + Use: "appsec-rules [appsec-rule]...", + Short: "Manage hub appsec rules", + Example: `cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs `, Args: cobra.MinimumNArgs(1), - Aliases: []string{"waap-rule"}, + Aliases: []string{"appsec-rule"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if _, err := require.Hub(csConfig, require.RemoteHub(csConfig)); err != nil { @@ -44,21 +44,21 @@ cscli waap-rules remove crowdsecurity/crs }, } - cmdWaapRules.AddCommand(NewCmdWaapRulesInstall()) - cmdWaapRules.AddCommand(NewCmdWaapRulesRemove()) - cmdWaapRules.AddCommand(NewCmdWaapRulesUpgrade()) - cmdWaapRules.AddCommand(NewCmdWaapRulesInspect()) - cmdWaapRules.AddCommand(NewCmdWaapRulesList()) + cmdAppsecRules.AddCommand(NewCmdAppsecRulesInstall()) + cmdAppsecRules.AddCommand(NewCmdAppsecRulesRemove()) + cmdAppsecRules.AddCommand(NewCmdAppsecRulesUpgrade()) + cmdAppsecRules.AddCommand(NewCmdAppsecRulesInspect()) + cmdAppsecRules.AddCommand(NewCmdAppsecRulesList()) - return cmdWaapRules + return cmdAppsecRules } -func NewCmdWaapRulesInstall() *cobra.Command { - cmdWaapRulesInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given waap rule(s)", - Long: `Fetch and install one or more waap rules from the hub`, - Example: `cscli waap-rules install crowdsecurity/crs`, +func NewCmdAppsecRulesInstall() *cobra.Command { + cmdAppsecRulesInstall := &cobra.Command{ + Use: "install ...", + Short: "Install given appsec rule(s)", + Long: `Fetch and install one or more appsec rules from the hub`, + Example: `cscli appsec-rules install crowdsecurity/crs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -67,20 +67,20 @@ func NewCmdWaapRulesInstall() *cobra.Command { RunE: itemsInstallRunner(hubItemTypes[cwhub.WAAP_RULES]), } - flags := cmdWaapRulesInstall.Flags() + flags := cmdAppsecRulesInstall.Flags() flags.BoolP("download-only", "d", false, "Only download packages, don't enable") flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple waap rules") + flags.Bool("ignore", false, "Ignore errors when installing multiple appsec rules") - return cmdWaapRulesInstall + return cmdAppsecRulesInstall } -func NewCmdWaapRulesRemove() *cobra.Command { - cmdWaapRulesRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given waap rule(s)", - Long: `remove one or more waap rules`, - Example: `cscli waap-rules remove crowdsecurity/crs`, +func NewCmdAppsecRulesRemove() *cobra.Command { + cmdAppsecRulesRemove := &cobra.Command{ + Use: "remove ...", + Short: "Remove given appsec rule(s)", + Long: `remove one or more appsec rules`, + Example: `cscli appsec-rules remove crowdsecurity/crs`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -89,20 +89,20 @@ func NewCmdWaapRulesRemove() *cobra.Command { RunE: itemsRemoveRunner(hubItemTypes[cwhub.WAAP_RULES]), } - flags := cmdWaapRulesRemove.Flags() + flags := cmdAppsecRulesRemove.Flags() flags.Bool("purge", false, "Delete source file too") flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the waap rules") + flags.Bool("all", false, "Remove all the appsec rules") - return cmdWaapRulesRemove + return cmdAppsecRulesRemove } -func NewCmdWaapRulesUpgrade() *cobra.Command { - cmdWaapRulesUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given waap rule(s)", - Long: `Fetch and upgrade one or more waap rules from the hub`, - Example: `cscli waap-rules upgrade crowdsecurity/crs`, +func NewCmdAppsecRulesUpgrade() *cobra.Command { + cmdAppsecRulesUpgrade := &cobra.Command{ + Use: "upgrade ...", + Short: "Upgrade given appsec rule(s)", + Long: `Fetch and upgrade one or more appsec rules from the hub`, + Example: `cscli appsec-rules upgrade crowdsecurity/crs`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) @@ -110,14 +110,14 @@ func NewCmdWaapRulesUpgrade() *cobra.Command { RunE: itemsUpgradeRunner(hubItemTypes[cwhub.WAAP_RULES]), } - flags := cmdWaapRulesUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the waap rules") + flags := cmdAppsecRulesUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the appsec rules") flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - return cmdWaapRulesUpgrade + return cmdAppsecRulesUpgrade } -func WaapRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { +func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { f := itemsInspectRunner(hubItemTypes[cwhub.WAAP_RULES]) if err := f(cmd, args); err != nil { @@ -127,19 +127,19 @@ func WaapRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args hub, _ := require.Hub(csConfig, nil) for _, name := range args { hubItem := hub.GetItem(itemType.name, name) - waapRule := waf.AppsecCollectionConfig{} + appsecRule := waf.AppsecCollectionConfig{} yamlContent, err := os.ReadFile(hubItem.State.LocalPath) if err != nil { return fmt.Errorf("unable to read file %s : %s", hubItem.State.LocalPath, err) } - if err := yaml.Unmarshal(yamlContent, &waapRule); err != nil { + if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { return fmt.Errorf("unable to unmarshal yaml file %s : %s", hubItem.State.LocalPath, err) } for _, ruleType := range waap_rule.SupportedTypes() { fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) - for _, rule := range waapRule.Rules { - convertedRule, _, err := rule.Convert(ruleType, waapRule.Name) + for _, rule := range appsecRule.Rules { + convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) if err != nil { return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) } @@ -152,42 +152,41 @@ func WaapRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args } } -func NewCmdWaapRulesInspect() *cobra.Command { - //FIXME; show the "compiled" rule - cmdWaapRulesInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a waap rule", - Long: `Inspect a waap rule`, - Example: `cscli waap-rules inspect crowdsecurity/crs`, +func NewCmdAppsecRulesInspect() *cobra.Command { + cmdAppsecRulesInspect := &cobra.Command{ + Use: "inspect ", + Short: "Inspect a appsec rule", + Long: `Inspect a appsec rule`, + Example: `cscli appsec-rules inspect crowdsecurity/crs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, - RunE: WaapRulesInspectRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: AppsecRulesInspectRunner(hubItemTypes[cwhub.WAAP_RULES]), } - flags := cmdWaapRulesInspect.Flags() + flags := cmdAppsecRulesInspect.Flags() flags.StringP("url", "u", "", "Prometheus url") flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - return cmdWaapRulesInspect + return cmdAppsecRulesInspect } -func NewCmdWaapRulesList() *cobra.Command { - cmdWaapRulesList := &cobra.Command{ - Use: "list [waap-rule]...", - Short: "List waap rules", - Long: `List of installed/available/specified waap rules`, - Example: `cscli waap-rules list -cscli waap-rules list -a -cscli waap-rules list crowdsecurity/crs`, +func NewCmdAppsecRulesList() *cobra.Command { + cmdAppsecRulesList := &cobra.Command{ + Use: "list [appsec-rule]...", + Short: "List appsec rules", + Long: `List of installed/available/specified appsec rules`, + Example: `cscli appsec-rules list +cscli appsec-rules list -a +cscli appsec-rules list crowdsecurity/crs`, DisableAutoGenTag: true, RunE: itemsListRunner(hubItemTypes[cwhub.WAAP_RULES]), } - flags := cmdWaapRulesList.Flags() + flags := cmdAppsecRulesList.Flags() flags.BoolP("all", "a", false, "List disabled items as well") - return cmdWaapRulesList + return cmdAppsecRulesList } diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 1fed1f78895..d429719b586 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -131,64 +131,64 @@ cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing List only enabled scenarios unless "-a" or names are specified.`, }, }, - "waap-rules": { - name: "waap-rules", - singular: "waap-rule", - oneOrMore: "waap-rule(s)", + "appsec-rules": { + name: "appsec-rules", + singular: "appsec-rule", + oneOrMore: "appsec-rule(s)", help: cmdHelp{ - example: `cscli waap-rules list -a -cscli waap-rules install crowdsecurity/crs -cscli waap-rules inspect crowdsecurity/crs -cscli waap-rules upgrade crowdsecurity/crs -cscli waap-rules remove crowdsecurity/crs + example: `cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs `, }, installHelp: cmdHelp{ - example: `cscli waap-rules install crowdsecurity/crs`, + example: `cscli appsec-rules install crowdsecurity/crs`, }, removeHelp: cmdHelp{ - example: `cscli waap-rules remove crowdsecurity/crs`, + example: `cscli appsec-rules remove crowdsecurity/crs`, }, upgradeHelp: cmdHelp{ - example: `cscli waap-rules upgrade crowdsecurity/crs`, + example: `cscli appsec-rules upgrade crowdsecurity/crs`, }, inspectHelp: cmdHelp{ - example: `cscli waap-rules inspect crowdsecurity/crs`, + example: `cscli appsec-rules inspect crowdsecurity/crs`, }, listHelp: cmdHelp{ - example: `cscli waap-rules list -cscli waap-rules list -a -cscli waap-rules list crowdsecurity/crs`, + example: `cscli appsec-rules list +cscli appsec-rules list -a +cscli appsec-rules list crowdsecurity/crs`, }, }, - "waap-configs": { - name: "waap-configs", - singular: "waap-config", - oneOrMore: "waap-config(s)", + "appsec-configs": { + name: "appsec-configs", + singular: "appsec-config", + oneOrMore: "appsec-config(s)", help: cmdHelp{ - example: `cscli waap-configs list -a -cscli waap-configs install crowdsecurity/vpatch -cscli waap-configs inspect crowdsecurity/vpatch -cscli waap-configs upgrade crowdsecurity/vpatch -cscli waap-configs remove crowdsecurity/vpatch + example: `cscli appsec-configs list -a +cscli appsec-configs install crowdsecurity/vpatch +cscli appsec-configs inspect crowdsecurity/vpatch +cscli appsec-configs upgrade crowdsecurity/vpatch +cscli appsec-configs remove crowdsecurity/vpatch `, }, installHelp: cmdHelp{ - example: `cscli waap-configs install crowdsecurity/vpatch`, + example: `cscli appsec-configs install crowdsecurity/vpatch`, }, removeHelp: cmdHelp{ - example: `cscli waap-configs remove crowdsecurity/vpatch`, + example: `cscli appsec-configs remove crowdsecurity/vpatch`, }, upgradeHelp: cmdHelp{ - example: `cscli waap-configs upgrade crowdsecurity/vpatch`, + example: `cscli appsec-configs upgrade crowdsecurity/vpatch`, }, inspectHelp: cmdHelp{ - example: `cscli waap-configs inspect crowdsecurity/vpatch`, + example: `cscli appsec-configs inspect crowdsecurity/vpatch`, }, listHelp: cmdHelp{ - example: `cscli waap-configs list -cscli waap-configs list -a -cscli waap-configs list crowdsecurity/vpatch`, + example: `cscli appsec-configs list +cscli appsec-configs list -a +cscli appsec-configs list crowdsecurity/vpatch`, }, }, "collections": { diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index ef7a0c0eee7..d5def40c7ed 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -242,12 +242,12 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - rootCmd.AddCommand(NewWaapRulesCmd()) // Keep it like this for now, we'll switch later to the generic implementation + rootCmd.AddCommand(NewAppsecRulesCmd()) // Keep it like this for now, we'll switch later to the generic implementation rootCmd.AddCommand(NewItemsCmd("collections")) rootCmd.AddCommand(NewItemsCmd("parsers")) rootCmd.AddCommand(NewItemsCmd("scenarios")) rootCmd.AddCommand(NewItemsCmd("postoverflows")) - rootCmd.AddCommand(NewItemsCmd("waap-configs")) + rootCmd.AddCommand(NewItemsCmd("appsec-configs")) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 0c147f932de..977d1bec0e1 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -53,8 +53,8 @@ type AppsecSource struct { addr string outChan chan types.Event InChan chan waf.ParsedRequest - AppsecRuntime *waf.WaapRuntimeConfig - AppsecConfigs map[string]waf.WaapConfig + AppsecRuntime *waf.AppsecRuntimeConfig + AppsecConfigs map[string]waf.AppsecConfig lapiURL string AuthCache AuthCache AppsecRunners []AppsecRunner //one for each go-routine @@ -170,7 +170,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { } w.InChan = make(chan waf.ParsedRequest) - appsecCfg := waf.WaapConfig{Logger: w.logger.WithField("component", "appsec_config")} + appsecCfg := waf.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} //let's load the associated appsec_config: if w.config.AppsecConfigPath != "" { From 2089ad66634d3595b37a03a5c5e5c53c9bb5fbcb Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 22:36:25 +0100 Subject: [PATCH 237/263] appsec renaming, part 4 --- cmd/crowdsec-cli/appsec_rules.go | 20 +-- cmd/crowdsec-cli/hubtest.go | 66 ++++---- cmd/crowdsec-cli/hubtest_table.go | 4 +- cmd/crowdsec-cli/item_metrics.go | 2 +- cmd/crowdsec-cli/metrics.go | 38 ++--- cmd/crowdsec-cli/metrics_table.go | 18 +- cmd/crowdsec/parse.go | 2 +- pkg/acquisition/modules/appsec/appsec.go | 12 +- .../modules/appsec/appsec_runner.go | 114 ++++++------- pkg/acquisition/modules/appsec/utils.go | 4 +- pkg/cwhub/item.go | 38 ++--- pkg/cwhub/sync.go | 8 +- pkg/hubtest/coverage.go | 34 ++-- pkg/hubtest/hubtest.go | 64 +++---- pkg/hubtest/hubtest_item.go | 160 +++++++++--------- pkg/hubtest/utils.go | 2 +- pkg/waf/loader.go | 6 +- pkg/waf/request.go | 16 +- pkg/waf/waap.go | 2 +- pkg/waf/waap_rules_collection.go | 2 +- 20 files changed, 306 insertions(+), 306 deletions(-) diff --git a/cmd/crowdsec-cli/appsec_rules.go b/cmd/crowdsec-cli/appsec_rules.go index 08aa34902b2..a5da34c3c4c 100644 --- a/cmd/crowdsec-cli/appsec_rules.go +++ b/cmd/crowdsec-cli/appsec_rules.go @@ -62,9 +62,9 @@ func NewCmdAppsecRulesInstall() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.WAAP_RULES, args, toComplete) + return compAllItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsInstallRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: itemsInstallRunner(hubItemTypes[cwhub.APPSEC_RULES]), } flags := cmdAppsecRulesInstall.Flags() @@ -84,9 +84,9 @@ func NewCmdAppsecRulesRemove() *cobra.Command { Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) + return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsRemoveRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: itemsRemoveRunner(hubItemTypes[cwhub.APPSEC_RULES]), } flags := cmdAppsecRulesRemove.Flags() @@ -105,9 +105,9 @@ func NewCmdAppsecRulesUpgrade() *cobra.Command { Example: `cscli appsec-rules upgrade crowdsecurity/crs`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) + return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsUpgradeRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: itemsUpgradeRunner(hubItemTypes[cwhub.APPSEC_RULES]), } flags := cmdAppsecRulesUpgrade.Flags() @@ -119,7 +119,7 @@ func NewCmdAppsecRulesUpgrade() *cobra.Command { func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - f := itemsInspectRunner(hubItemTypes[cwhub.WAAP_RULES]) + f := itemsInspectRunner(hubItemTypes[cwhub.APPSEC_RULES]) if err := f(cmd, args); err != nil { return err } @@ -161,9 +161,9 @@ func NewCmdAppsecRulesInspect() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) + return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: AppsecRulesInspectRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: AppsecRulesInspectRunner(hubItemTypes[cwhub.APPSEC_RULES]), } flags := cmdAppsecRulesInspect.Flags() @@ -182,7 +182,7 @@ func NewCmdAppsecRulesList() *cobra.Command { cscli appsec-rules list -a cscli appsec-rules list crowdsecurity/crs`, DisableAutoGenTag: true, - RunE: itemsListRunner(hubItemTypes[cwhub.WAAP_RULES]), + RunE: itemsListRunner(hubItemTypes[cwhub.APPSEC_RULES]), } flags := cmdAppsecRulesList.Flags() diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 5ff15af7095..295a8f89dae 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -19,9 +19,9 @@ import ( ) var HubTest hubtest.HubTest -var HubWaapTests hubtest.HubTest +var HubAppsecTests hubtest.HubTest var hubPtr *hubtest.HubTest -var isWaapTest bool +var isAppsecTest bool func NewHubTestCmd() *cobra.Command { var hubPath string @@ -41,14 +41,14 @@ func NewHubTestCmd() *cobra.Command { return fmt.Errorf("unable to load hubtest: %+v", err) } - HubWaapTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true) + HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true) if err != nil { - return fmt.Errorf("unable to load waap specific hubtest: %+v", err) + return fmt.Errorf("unable to load appsec specific hubtest: %+v", err) } - /*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to WAAP tests*/ + /*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/ hubPtr = &HubTest - if isWaapTest { - hubPtr = &HubWaapTests + if isAppsecTest { + hubPtr = &HubAppsecTests } return nil }, @@ -57,7 +57,7 @@ func NewHubTestCmd() *cobra.Command { cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") - cmdHubTest.PersistentFlags().BoolVar(&isWaapTest, "waap", false, "Command relates to WAAP tests") + cmdHubTest.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests") cmdHubTest.AddCommand(NewHubTestCreateCmd()) cmdHubTest.AddCommand(NewHubTestRunCmd()) @@ -105,7 +105,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios configFilePath := filepath.Join(testPath, "config.yaml") configFileData := &hubtest.HubTestItemConfig{} - if logType == "waap" { + if logType == "appsec" { //create empty nuclei template file nucleiFileName := fmt.Sprintf("%s.yaml", testName) nucleiFilePath := filepath.Join(testPath, nucleiFileName) @@ -114,7 +114,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return err } nucleiFile.Close() - configFileData.WaapRules = []string{"your_rule_here.yaml"} + configFileData.AppsecRules = []string{"your_rule_here.yaml"} configFileData.NucleiTemplate = nucleiFileName fmt.Println() fmt.Printf(" Test name : %s\n", testName) @@ -246,7 +246,7 @@ func NewHubTestRunCmd() *cobra.Command { success := true testResult := make(map[string]bool) for _, test := range hubPtr.Tests { - if test.AutoGen && !isWaapTest { + if test.AutoGen && !isAppsecTest { if test.ParserAssert.AutoGenAssert { log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File) fmt.Println() @@ -400,9 +400,9 @@ func NewHubTestInfoCmd() *cobra.Command { fmt.Println() fmt.Printf(" Test name : %s\n", test.Name) fmt.Printf(" Test path : %s\n", test.Path) - if isWaapTest { + if isAppsecTest { fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate) - fmt.Printf(" Waap Rules : %s\n", strings.Join(test.Config.WaapRules, ", ")) + fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", ")) } else { fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile)) fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName)) @@ -452,27 +452,27 @@ func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool var showOnlyPercent bool - var showWaapCov bool + var showAppsecCov bool var cmdHubTestCoverage = &cobra.Command{ Use: "coverage", Short: "coverage", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - //for this one we explictely don't do for Waap + //for this one we explicitly don't do for appsec if err := HubTest.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %+v", err) } var err error scenarioCoverage := []hubtest.Coverage{} parserCoverage := []hubtest.Coverage{} - waapRuleCoverage := []hubtest.Coverage{} + appsecRuleCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 - waapRuleCoveragePercent := 0 + appsecRuleCoveragePercent := 0 // if both are false (flag by default), show both - showAll := !showScenarioCov && !showParserCov && !showWaapCov + showAll := !showScenarioCov && !showParserCov && !showAppsecCov if showParserCov || showAll { parserCoverage, err = HubTest.GetParsersCoverage() @@ -504,30 +504,30 @@ func NewHubTestCoverageCmd() *cobra.Command { scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } - if showWaapCov || showAll { - waapRuleCoverage, err = HubTest.GetWaapCoverage() + if showAppsecCov || showAll { + appsecRuleCoverage, err = HubTest.GetAppsecCoverage() if err != nil { return fmt.Errorf("while getting scenario coverage: %s", err) } - waapRuleTested := 0 - for _, test := range waapRuleCoverage { + appsecRuleTested := 0 + for _, test := range appsecRuleCoverage { if test.TestsCount > 0 { - waapRuleTested++ + appsecRuleTested++ } } - waapRuleCoveragePercent = int(math.Round((float64(waapRuleTested) / float64(len(waapRuleCoverage)) * 100))) + appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100))) } if showOnlyPercent { if showAll { - fmt.Printf("parsers=%d%%\nscenarios=%d%%\nwaap_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, waapRuleCoveragePercent) + fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent) } else if showParserCov { fmt.Printf("parsers=%d%%", parserCoveragePercent) } else if showScenarioCov { fmt.Printf("scenarios=%d%%", scenarioCoveragePercent) - } else if showWaapCov { - fmt.Printf("waap_rules=%d%%", waapRuleCoveragePercent) + } else if showAppsecCov { + fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent) } os.Exit(0) } @@ -542,8 +542,8 @@ func NewHubTestCoverageCmd() *cobra.Command { hubTestScenarioCoverageTable(color.Output, scenarioCoverage) } - if showWaapCov || showAll { - hubTestWaapRuleCoverageTable(color.Output, waapRuleCoverage) + if showAppsecCov || showAll { + hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage) } fmt.Println() @@ -553,8 +553,8 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } - if showWaapCov || showAll { - fmt.Printf("WAAP RULES : %d%% of coverage\n", waapRuleCoveragePercent) + if showAppsecCov || showAll { + fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent) } case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") @@ -567,7 +567,7 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) - dump, err = json.MarshalIndent(waapRuleCoverage, "", " ") + dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ") if err != nil { return err } @@ -583,7 +583,7 @@ func NewHubTestCoverageCmd() *cobra.Command { cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") - cmdHubTestCoverage.PersistentFlags().BoolVar(&showWaapCov, "waap", false, "Show only waap coverage") + cmdHubTestCoverage.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage") return cmdHubTestCoverage } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 4a24d8d1715..4034da7e519 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -61,9 +61,9 @@ func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t.Render() } -func hubTestWaapRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) { +func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) - t.SetHeaders("Waap Rule", "Status", "Number of tests") + t.SetHeaders("Appsec Rule", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index fb138141b8f..34484f63d5d 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -32,7 +32,7 @@ func ShowMetrics(hubItem *cwhub.Item) error { return err } } - case cwhub.WAAP_RULES: + case cwhub.APPSEC_RULES: log.Error("FIXME: not implemented yet") default: // no metrics for this item type diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index c4fcb71c167..20cbed6dfe2 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -63,8 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error lapi_machine_stats := map[string]map[string]map[string]int{} lapi_bouncer_stats := map[string]map[string]map[string]int{} decisions_stats := map[string]map[string]map[string]int{} - waap_engine_stats := map[string]map[string]int{} - waap_rule_stats := map[string]map[string]map[string]int{} + appsec_engine_stats := map[string]map[string]int{} + appsec_rule_stats := map[string]map[string]map[string]int{} alerts_stats := map[string]int{} stash_stats := map[string]struct { Type string @@ -228,26 +228,26 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error Type string Count int }{Type: mtype, Count: ival} - case "cs_waf_reqs_total": - if _, ok := waap_engine_stats[metric.Labels["waap_engine"]]; !ok { - waap_engine_stats[metric.Labels["waap_engine"]] = make(map[string]int, 0) + case "cs_appsec_reqs_total": + if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok { + appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0) } - waap_engine_stats[metric.Labels["waap_engine"]]["processed"] = ival - case "cs_waf_block_total": - if _, ok := waap_engine_stats[metric.Labels["waap_engine"]]; !ok { - waap_engine_stats[metric.Labels["waap_engine"]] = make(map[string]int, 0) + appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival + case "cs_appsec_block_total": + if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok { + appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0) } - waap_engine_stats[metric.Labels["waap_engine"]]["blocked"] = ival - case "cs_waf_rule_hits": - waapEngine := metric.Labels["waap_engine"] + appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival + case "cs_appsec_rule_hits": + appsecEngine := metric.Labels["appsec_engine"] ruleID := metric.Labels["rule_name"] - if _, ok := waap_rule_stats[waapEngine]; !ok { - waap_rule_stats[waapEngine] = make(map[string]map[string]int, 0) + if _, ok := appsec_rule_stats[appsecEngine]; !ok { + appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0) } - if _, ok := waap_rule_stats[waapEngine][ruleID]; !ok { - waap_rule_stats[waapEngine][ruleID] = make(map[string]int, 0) + if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok { + appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0) } - waap_rule_stats[waapEngine][ruleID]["triggered"] = ival + appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival default: log.Infof("unknown: %+v", fam.Name) continue @@ -266,8 +266,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error decisionStatsTable(out, decisions_stats) alertStatsTable(out, alerts_stats) stashStatsTable(out, stash_stats) - waapMetricsToTable(out, waap_engine_stats) - waapRulesToTable(out, waap_rule_stats) + appsecMetricsToTable(out, appsec_engine_stats) + appsecRulesToTable(out, appsec_rule_stats) return nil } diff --git a/cmd/crowdsec-cli/metrics_table.go b/cmd/crowdsec-cli/metrics_table.go index 7df03318cfb..80b9cb6e435 100644 --- a/cmd/crowdsec-cli/metrics_table.go +++ b/cmd/crowdsec-cli/metrics_table.go @@ -113,31 +113,31 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) { } } -func waapMetricsToTable(out io.Writer, metrics map[string]map[string]int) { +func appsecMetricsToTable(out io.Writer, metrics map[string]map[string]int) { t := newTable(out) t.SetRowLines(false) - t.SetHeaders("WAF Engine", "Processed", "Blocked") + t.SetHeaders("Appsec Engine", "Processed", "Blocked") t.SetAlignment(table.AlignLeft, table.AlignLeft) keys := []string{"processed", "blocked"} if numRows, err := metricsToTable(t, metrics, keys); err != nil { - log.Warningf("while collecting waap stats: %s", err) + log.Warningf("while collecting appsec stats: %s", err) } else if numRows > 0 { - renderTableTitle(out, "\nWaap Metrics:") + renderTableTitle(out, "\nAppsec Metrics:") t.Render() } } -func waapRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) { - for waapEngine, waapEngineRulesStats := range metrics { +func appsecRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) { + for appsecEngine, appsecEngineRulesStats := range metrics { t := newTable(out) t.SetRowLines(false) t.SetHeaders("Rule ID", "Triggered") t.SetAlignment(table.AlignLeft, table.AlignLeft) keys := []string{"triggered"} - if numRows, err := metricsToTable(t, waapEngineRulesStats, keys); err != nil { - log.Warningf("while collecting waap rules stats: %s", err) + if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys); err != nil { + log.Warningf("while collecting appsec rules stats: %s", err) } else if numRows > 0 { - renderTableTitle(out, fmt.Sprintf("\nWaap '%s' Rules Metrics:", waapEngine)) + renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine)) t.Render() } } diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index e1945b027dd..c62eeb5869d 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -22,7 +22,7 @@ LOOP: if !event.Process { continue } - /*Waap is going to generate 2 events: + /*Application security engine is going to generate 2 events: - one that is treated as a log and can go to scenarios - another one that will go directly to LAPI*/ if event.Type == types.APPSEC { diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 977d1bec0e1..e128ba6aec5 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -31,7 +31,7 @@ var ( DefaultAuthCacheDuration = (1 * time.Minute) ) -// configuration structure of the acquis for the Waap +// configuration structure of the acquis for the application security engine type AppsecSourceConfig struct { ListenAddr string `yaml:"listen_addr"` CertFilePath string `yaml:"cert_file"` @@ -44,7 +44,7 @@ type AppsecSourceConfig struct { configuration.DataSourceCommonCfg `yaml:",inline"` } -// runtime structure of WaapSourceConfig +// runtime structure of AppsecSourceConfig type AppsecSource struct { config AppsecSourceConfig logger *log.Entry @@ -210,7 +210,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { logger: w.logger.WithFields(log.Fields{ "uuid": appsecRunnerUUID, }), - WaapRuntime: &wrt, + AppsecRuntime: &wrt, } err := runner.Init(appsecCfg.GetDataDir()) if err != nil { @@ -345,15 +345,15 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusInternalServerError) return } - parsedRequest.WaapEngine = w.config.Name + parsedRequest.AppsecEngine = w.config.Name - AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.WaapEngine}).Inc() + AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() w.InChan <- parsedRequest response := <-parsedRequest.ResponseChannel if response.InBandInterrupt { - AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.WaapEngine}).Inc() + AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() } appsecResponse := w.AppsecRuntime.GenerateResponse(response) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 0aeca248990..6ce7e57df40 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -15,15 +15,15 @@ import ( "gopkg.in/tomb.v2" ) -// that's the runtime structure of the WAAP as seen from the acquis +// that's the runtime structure of the Application security engine as seen from the acquis type AppsecRunner struct { - outChan chan types.Event - inChan chan waf.ParsedRequest - UUID string - WaapRuntime *waf.AppsecRuntimeConfig //this holds the actual waap runtime config, rules, remediations, hooks etc. - WaapInbandEngine coraza.WAF - WaapOutbandEngine coraza.WAF - logger *log.Entry + outChan chan types.Event + inChan chan waf.ParsedRequest + UUID string + AppsecRuntime *waf.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc. + AppsecInbandEngine coraza.WAF + AppsecOutbandEngine coraza.WAF + logger *log.Entry } func (r *AppsecRunner) Init(datadir string) error { @@ -33,11 +33,11 @@ func (r *AppsecRunner) Init(datadir string) error { inBandRules := "" outOfBandRules := "" - for _, collection := range r.WaapRuntime.InBandRules { + for _, collection := range r.AppsecRuntime.InBandRules { inBandRules += collection.String() } - for _, collection := range r.WaapRuntime.OutOfBandRules { + for _, collection := range r.AppsecRuntime.OutOfBandRules { outOfBandRules += collection.String() } inBandLogger := r.logger.Dup().WithField("band", "inband") @@ -45,52 +45,52 @@ func (r *AppsecRunner) Init(datadir string) error { //setting up inband engine inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(inBandLogger)) - if !r.WaapRuntime.Config.InbandOptions.DisableBodyInspection { + if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection { inbandCfg = inbandCfg.WithRequestBodyAccess() } else { log.Warningf("Disabling body inspection, Inband rules will not be able to match on body's content.") } - if r.WaapRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil { - inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.WaapRuntime.Config.InbandOptions.RequestBodyInMemoryLimit) + if r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil { + inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit) } - r.WaapInbandEngine, err = coraza.NewWAF(inbandCfg) + r.AppsecInbandEngine, err = coraza.NewWAF(inbandCfg) if err != nil { return fmt.Errorf("unable to initialize inband engine : %w", err) } //setting up outband engine outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(outBandLogger)) - if !r.WaapRuntime.Config.OutOfBandOptions.DisableBodyInspection { + if !r.AppsecRuntime.Config.OutOfBandOptions.DisableBodyInspection { outbandCfg = outbandCfg.WithRequestBodyAccess() } else { log.Warningf("Disabling body inspection, Out of band rules will not be able to match on body's content.") } - if r.WaapRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil { - outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.WaapRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit) + if r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil { + outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit) } - r.WaapOutbandEngine, err = coraza.NewWAF(outbandCfg) + r.AppsecOutbandEngine, err = coraza.NewWAF(outbandCfg) - if r.WaapRuntime.DisabledInBandRulesTags != nil { - for _, tag := range r.WaapRuntime.DisabledInBandRulesTags { - r.WaapInbandEngine.GetRuleGroup().DeleteByTag(tag) + if r.AppsecRuntime.DisabledInBandRulesTags != nil { + for _, tag := range r.AppsecRuntime.DisabledInBandRulesTags { + r.AppsecInbandEngine.GetRuleGroup().DeleteByTag(tag) } } - if r.WaapRuntime.DisabledOutOfBandRulesTags != nil { - for _, tag := range r.WaapRuntime.DisabledOutOfBandRulesTags { - r.WaapOutbandEngine.GetRuleGroup().DeleteByTag(tag) + if r.AppsecRuntime.DisabledOutOfBandRulesTags != nil { + for _, tag := range r.AppsecRuntime.DisabledOutOfBandRulesTags { + r.AppsecOutbandEngine.GetRuleGroup().DeleteByTag(tag) } } - if r.WaapRuntime.DisabledInBandRuleIds != nil { - for _, id := range r.WaapRuntime.DisabledInBandRuleIds { - r.WaapInbandEngine.GetRuleGroup().DeleteByID(id) + if r.AppsecRuntime.DisabledInBandRuleIds != nil { + for _, id := range r.AppsecRuntime.DisabledInBandRuleIds { + r.AppsecInbandEngine.GetRuleGroup().DeleteByID(id) } } - if r.WaapRuntime.DisabledOutOfBandRuleIds != nil { - for _, id := range r.WaapRuntime.DisabledOutOfBandRuleIds { - r.WaapOutbandEngine.GetRuleGroup().DeleteByID(id) + if r.AppsecRuntime.DisabledOutOfBandRuleIds != nil { + for _, id := range r.AppsecRuntime.DisabledOutOfBandRuleIds { + r.AppsecOutbandEngine.GetRuleGroup().DeleteByID(id) } } @@ -117,7 +117,7 @@ func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.P }() //pre eval (expr) rules - err = r.WaapRuntime.ProcessPreEvalRules(request) + err = r.AppsecRuntime.ProcessPreEvalRules(request) if err != nil { r.logger.Errorf("unable to process PreEval rules: %s", err) //FIXME: should we abort here ? @@ -177,7 +177,7 @@ func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.P r.logger.Debugf("rules matched for body : %d", in.RuleID) } - err = r.WaapRuntime.ProcessPostEvalRules(request) + err = r.AppsecRuntime.ProcessPostEvalRules(request) if err != nil { r.logger.Errorf("unable to process PostEval rules: %s", err) } @@ -186,15 +186,15 @@ func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.P } func (r *AppsecRunner) ProcessInBandRules(request *waf.ParsedRequest) error { - tx := waf.NewExtendedTransaction(r.WaapInbandEngine, request.UUID) - r.WaapRuntime.InBandTx = tx + tx := waf.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) + r.AppsecRuntime.InBandTx = tx err := r.processRequest(tx, request) return err } func (r *AppsecRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { - tx := waf.NewExtendedTransaction(r.WaapOutbandEngine, request.UUID) - r.WaapRuntime.OutOfBandTx = tx + tx := waf.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) + r.AppsecRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) return err } @@ -212,32 +212,32 @@ func (r *AppsecRunner) handleInBandInterrupt(request *waf.ParsedRequest) { } if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("inband rules matched : %d", in.RuleID) - r.WaapRuntime.Response.InBandInterrupt = true - r.WaapRuntime.Response.HTTPResponseCode = r.WaapRuntime.Config.BlockedHTTPCode - r.WaapRuntime.Response.Action = r.WaapRuntime.DefaultRemediation + r.AppsecRuntime.Response.InBandInterrupt = true + r.AppsecRuntime.Response.HTTPResponseCode = r.AppsecRuntime.Config.BlockedHTTPCode + r.AppsecRuntime.Response.Action = r.AppsecRuntime.DefaultRemediation - if _, ok := r.WaapRuntime.RemediationById[in.RuleID]; ok { - r.WaapRuntime.Response.Action = r.WaapRuntime.RemediationById[in.RuleID] + if _, ok := r.AppsecRuntime.RemediationById[in.RuleID]; ok { + r.AppsecRuntime.Response.Action = r.AppsecRuntime.RemediationById[in.RuleID] } - for tag, remediation := range r.WaapRuntime.RemediationByTag { + for tag, remediation := range r.AppsecRuntime.RemediationByTag { if slices.Contains[[]string, string](in.Tags, tag) { - r.WaapRuntime.Response.Action = remediation + r.AppsecRuntime.Response.Action = remediation } } - err = r.WaapRuntime.ProcessOnMatchRules(request, evt) + err = r.AppsecRuntime.ProcessOnMatchRules(request, evt) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) return } // Should the in band match trigger an event ? - if r.WaapRuntime.Response.SendEvent { + if r.AppsecRuntime.Response.SendEvent { r.outChan <- evt } // Should the in band match trigger an overflow ? - if r.WaapRuntime.Response.SendAlert { + if r.AppsecRuntime.Response.SendAlert { appsecOvlfw, err := AppsecEventGeneration(evt) if err != nil { r.logger.Errorf("unable to generate appsec event : %s", err) @@ -260,23 +260,23 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { } if in := request.Tx.Interruption(); in != nil { r.logger.Debugf("inband rules matched : %d", in.RuleID) - r.WaapRuntime.Response.OutOfBandInterrupt = true + r.AppsecRuntime.Response.OutOfBandInterrupt = true - err = r.WaapRuntime.ProcessOnMatchRules(request, evt) + err = r.AppsecRuntime.ProcessOnMatchRules(request, evt) if err != nil { r.logger.Errorf("unable to process OnMatch rules: %s", err) return } // Should the match trigger an event ? - if r.WaapRuntime.Response.SendEvent { + if r.AppsecRuntime.Response.SendEvent { r.outChan <- evt } // Should the match trigger an overflow ? - if r.WaapRuntime.Response.SendAlert { + if r.AppsecRuntime.Response.SendAlert { appsecOvlfw, err := AppsecEventGeneration(evt) if err != nil { - r.logger.Errorf("unable to generate waap event : %s", err) + r.logger.Errorf("unable to generate appsec event : %s", err) return } r.outChan <- *appsecOvlfw @@ -286,7 +286,7 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { r.logger.Debugf("Requests handled by runner %s", request.UUID) - r.WaapRuntime.ClearResponse() + r.AppsecRuntime.ClearResponse() request.IsInBand = true request.IsOutBand = false @@ -294,7 +294,7 @@ func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { //to measure the time spent in the Application Security Engine startParsing := time.Now() - //inband WAAP rules + //inband appsec rules err := r.ProcessInBandRules(request) if err != nil { r.logger.Errorf("unable to process InBand rules: %s", err) @@ -309,14 +309,14 @@ func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) // send back the result to the HTTP handler for the InBand part - request.ResponseChannel <- r.WaapRuntime.Response + request.ResponseChannel <- r.AppsecRuntime.Response //Now let's process the out of band rules request.IsInBand = false request.IsOutBand = true - r.WaapRuntime.Response.SendAlert = false - r.WaapRuntime.Response.SendEvent = true + r.AppsecRuntime.Response.SendAlert = false + r.AppsecRuntime.Response.SendEvent = true err = r.ProcessOutOfBandRules(request) if err != nil { @@ -330,7 +330,7 @@ func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { } func (r *AppsecRunner) Run(t *tomb.Tomb) error { - r.logger.Infof("Waap Runner ready to process event") + r.logger.Infof("Appsec Runner ready to process event") for { select { case <-t.Dying(): diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index ea6c1670e29..a228cf72d1a 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -175,7 +175,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequ if variable.Value() == "" { continue } - for _, collectionToKeep := range r.WaapRuntime.CompiledVariablesTracking { + for _, collectionToKeep := range r.AppsecRuntime.CompiledVariablesTracking { match := collectionToKeep.MatchString(key) if match { evt.Waap.Vars[key] = variable.Value() @@ -215,7 +215,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequ r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) } - AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.WaapEngine}).Inc() + AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc() corazaRule := map[string]interface{}{ "id": rule.Rule().ID(), diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index b94a8573b99..3772d47680f 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -12,12 +12,12 @@ import ( const ( // managed item types. - COLLECTIONS = "collections" - PARSERS = "parsers" - POSTOVERFLOWS = "postoverflows" - SCENARIOS = "scenarios" - WAAP_CONFIGS = "waap-configs" - WAAP_RULES = "waap-rules" + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" + APPSEC_CONFIGS = "appsec-configs" + APPSEC_RULES = "appsec-rules" ) const ( @@ -29,7 +29,7 @@ const ( var ( // The order is important, as it is used to range over sub-items in collections. - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, WAAP_CONFIGS, WAAP_RULES, COLLECTIONS} + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS} ) type HubItems map[string]map[string]*Item @@ -78,8 +78,8 @@ type Item struct { PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` - WaapConfigs []string `json:"waap-configs,omitempty" yaml:"waap-configs,omitempty"` - WaapRules []string `json:"waap-rules,omitempty" yaml:"waap-rules,omitempty"` + AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` + AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` } // installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local @@ -194,8 +194,8 @@ func (i *Item) SubItems() []*Item { sub = append(sub, s) } - for _, name := range i.WaapConfigs { - s := i.hub.GetItem(WAAP_CONFIGS, name) + for _, name := range i.AppsecConfigs { + s := i.hub.GetItem(APPSEC_CONFIGS, name) if s == nil { continue } @@ -203,8 +203,8 @@ func (i *Item) SubItems() []*Item { sub = append(sub, s) } - for _, name := range i.WaapRules { - s := i.hub.GetItem(WAAP_RULES, name) + for _, name := range i.AppsecRules { + s := i.hub.GetItem(APPSEC_RULES, name) if s == nil { continue } @@ -247,15 +247,15 @@ func (i *Item) logMissingSubItems() { } } - for _, subName := range i.WaapConfigs { - if i.hub.GetItem(WAAP_CONFIGS, subName) == nil { - log.Errorf("can't find %s in %s, required by %s", subName, WAAP_CONFIGS, i.Name) + for _, subName := range i.AppsecConfigs { + if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) } } - for _, subName := range i.WaapRules { - if i.hub.GetItem(WAAP_RULES, subName) == nil { - log.Errorf("can't find %s in %s, required by %s", subName, WAAP_RULES, i.Name) + for _, subName := range i.AppsecRules { + if i.hub.GetItem(APPSEC_RULES, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) } } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 2a495716bbd..a5f839f0b00 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -118,11 +118,11 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { } else if ret.stage == COLLECTIONS { ret.ftype = COLLECTIONS ret.stage = "" - } else if ret.stage == WAAP_RULES { - ret.ftype = WAAP_RULES + } else if ret.stage == APPSEC_RULES { + ret.ftype = APPSEC_RULES ret.stage = "" - } else if ret.stage == WAAP_CONFIGS { - ret.ftype = WAAP_CONFIGS + } else if ret.stage == APPSEC_CONFIGS { + ret.ftype = APPSEC_CONFIGS ret.stage = "" } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { // it's a PARSER / POSTOVERFLOW with a stage diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index ad7ceff635f..ce4083573da 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -19,13 +19,13 @@ type Coverage struct { PresentIn map[string]bool //poorman's set } -func (h *HubTest) GetWaapCoverage() ([]Coverage, error) { - if len(h.HubIndex.GetItemMap(cwhub.WAAP_RULES)) == 0 { - return nil, fmt.Errorf("no waap rules in hub index") +func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { + if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 { + return nil, fmt.Errorf("no appsec rules in hub index") } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.WAAP_RULES)) + pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { @@ -37,16 +37,16 @@ func (h *HubTest) GetWaapCoverage() ([]Coverage, error) { } // parser the expressions a-la-oneagain - waapTestConfigs, err := filepath.Glob(".waap-tests/*/config.yaml") + appsecTestConfigs, err := filepath.Glob(".appsec-tests/*/config.yaml") if err != nil { - return nil, fmt.Errorf("while find waap-tests config: %s", err) + return nil, fmt.Errorf("while find appsec-tests config: %s", err) } - for _, waapTestConfigPath := range waapTestConfigs { + for _, appsecTestConfigPath := range appsecTestConfigs { configFileData := &HubTestItemConfig{} - yamlFile, err := os.ReadFile(waapTestConfigPath) + yamlFile, err := os.ReadFile(appsecTestConfigPath) if err != nil { - log.Printf("unable to open waap test config file '%s': %s", waapTestConfigPath, err) + log.Printf("unable to open appsec test config file '%s': %s", appsecTestConfigPath, err) continue } err = yaml.Unmarshal(yamlFile, configFileData) @@ -54,22 +54,22 @@ func (h *HubTest) GetWaapCoverage() ([]Coverage, error) { return nil, fmt.Errorf("unmarshal: %v", err) } - for _, waapRulesFile := range configFileData.WaapRules { - waapRuleData := &waap_rule.CustomRule{} - yamlFile, err := os.ReadFile(waapRulesFile) + for _, appsecRulesFile := range configFileData.AppsecRules { + appsecRuleData := &waap_rule.CustomRule{} + yamlFile, err := os.ReadFile(appsecRulesFile) if err != nil { - log.Printf("unable to open waap rule '%s': %s", waapRulesFile, err) + log.Printf("unable to open appsec rule '%s': %s", appsecRulesFile, err) } - err = yaml.Unmarshal(yamlFile, waapRuleData) + err = yaml.Unmarshal(yamlFile, appsecRuleData) if err != nil { return nil, fmt.Errorf("unmarshal: %v", err) } - waapRuleName := waapRuleData.Name + appsecRuleName := appsecRuleData.Name for idx, cov := range coverage { - if cov.Name == waapRuleName { + if cov.Name == appsecRuleName { coverage[idx].TestsCount++ - coverage[idx].PresentIn[waapTestConfigPath] = true + coverage[idx].PresentIn[appsecTestConfigPath] = true } } } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index fa25282b3ec..b9bde4e2c7d 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -11,30 +11,30 @@ import ( ) type HubTest struct { - CrowdSecPath string - CscliPath string - HubPath string - HubTestPath string //generic parser/scenario tests .tests - HubWaapTestPath string //dir specific to waap tests .waap-tests - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - TemplateAcquisPath string - TemplateWaapProfilePath string - HubIndex *cwhub.Hub - Tests []*HubTestItem + CrowdSecPath string + CscliPath string + HubPath string + HubTestPath string //generic parser/scenario tests .tests + HubAppsecTestPath string //dir specific to appsec tests .appsec-tests + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateAppsecProfilePath string + HubIndex *cwhub.Hub + Tests []*HubTestItem } const ( - templateConfigFile = "template_config.yaml" - templateSimulationFile = "template_simulation.yaml" - templateProfileFile = "template_profiles.yaml" - templateAcquisFile = "template_acquis.yaml" - templateWaapProfilePath = "template_waap-profile.yaml" + templateConfigFile = "template_config.yaml" + templateSimulationFile = "template_simulation.yaml" + templateProfileFile = "template_profiles.yaml" + templateAcquisFile = "template_acquis.yaml" + templateAppsecProfilePath = "template_appsec-profile.yaml" ) -func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isWaapTest bool) (HubTest, error) { +func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) @@ -58,8 +58,8 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isWaapTes } } - if isWaapTest { - HubTestPath := filepath.Join(hubPath, "./.waap-tests/") + if isAppsecTest { + HubTestPath := filepath.Join(hubPath, "./.appsec-tests/") hubIndexFile := filepath.Join(hubPath, ".index.json") local := &csconfig.LocalHubCfg{ @@ -75,17 +75,17 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isWaapTes } return HubTest{ - CrowdSecPath: crowdsecPath, - CscliPath: cscliPath, - HubPath: hubPath, - HubTestPath: HubTestPath, - HubIndexFile: hubIndexFile, - TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), - TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), - TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), - TemplateWaapProfilePath: filepath.Join(HubTestPath, templateWaapProfilePath), - TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), - HubIndex: hub, + CrowdSecPath: crowdsecPath, + CscliPath: cscliPath, + HubPath: hubPath, + HubTestPath: HubTestPath, + HubIndexFile: hubIndexFile, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), + TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath), + TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), + HubIndex: hub, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 8e0bc1a201f..14a022a6d55 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -21,7 +21,7 @@ type HubTestItemConfig struct { Parsers []string `yaml:"parsers,omitempty"` Scenarios []string `yaml:"scenarios,omitempty"` PostOverflows []string `yaml:"postoverflows,omitempty"` - WaapRules []string `yaml:"waap-rules,omitempty"` + AppsecRules []string `yaml:"appsec-rules,omitempty"` NucleiTemplate string `yaml:"nuclei_template,omitempty"` ExpectedNucleiFailure bool `yaml:"expect_failure,omitempty"` LogFile string `yaml:"log_file,omitempty"` @@ -53,15 +53,15 @@ type HubTestItem struct { ScenarioResultFile string BucketPourResultFile string - HubPath string - HubTestPath string - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - TemplateAcquisPath string - TemplateWaapProfilePath string - HubIndex *cwhub.Hub + HubPath string + HubTestPath string + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateAppsecProfilePath string + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -87,7 +87,7 @@ const ( TestBouncerApiKey = "this_is_a_bad_password" DefaultNucleiTarget = "http://127.0.0.1:80/" - DefaultWaapHost = "127.0.0.1:4241" + DefaultAppsecHost = "127.0.0.1:4241" ) func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { @@ -139,19 +139,19 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { InstallDir: runtimeFolder, InstallDataDir: filepath.Join(runtimeFolder, "data"), }, - Config: configFileData, - HubPath: hubTest.HubPath, - HubTestPath: hubTest.HubTestPath, - HubIndexFile: hubTest.HubIndexFile, - TemplateConfigPath: hubTest.TemplateConfigPath, - TemplateProfilePath: hubTest.TemplateProfilePath, - TemplateSimulationPath: hubTest.TemplateSimulationPath, - TemplateAcquisPath: hubTest.TemplateAcquisPath, - TemplateWaapProfilePath: hubTest.TemplateWaapProfilePath, - HubIndex: hubTest.HubIndex, - ScenarioAssert: ScenarioAssert, - ParserAssert: ParserAssert, - CustomItemsLocation: []string{hubTest.HubPath, testPath}, + Config: configFileData, + HubPath: hubTest.HubPath, + HubTestPath: hubTest.HubTestPath, + HubIndexFile: hubTest.HubIndexFile, + TemplateConfigPath: hubTest.TemplateConfigPath, + TemplateProfilePath: hubTest.TemplateProfilePath, + TemplateSimulationPath: hubTest.TemplateSimulationPath, + TemplateAcquisPath: hubTest.TemplateAcquisPath, + TemplateAppsecProfilePath: hubTest.TemplateAppsecProfilePath, + HubIndex: hubTest.HubIndex, + ScenarioAssert: ScenarioAssert, + ParserAssert: ParserAssert, + CustomItemsLocation: []string{hubTest.HubPath, testPath}, }, nil } @@ -313,75 +313,75 @@ func (t *HubTestItem) InstallHub() error { } } - // install waaprules in runtime environment - for _, waaprule := range t.Config.WaapRules { - log.Infof("adding rule '%s'", waaprule) - if waaprule == "" { + // install appsec-rules in runtime environment + for _, appsecrule := range t.Config.AppsecRules { + log.Infof("adding rule '%s'", appsecrule) + if appsecrule == "" { continue } - if hubWaapRule, ok := t.HubIndex.GetItemMap(cwhub.WAAP_RULES)[waaprule]; ok { - waapRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubWaapRule.RemotePath)) + if hubAppsecRule, ok := t.HubIndex.GetItemMap(cwhub.APPSEC_RULES)[appsecrule]; ok { + appsecRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubAppsecRule.RemotePath)) if err != nil { - return fmt.Errorf("can't get absolute path of '%s': %s", waapRuleSource, err) + return fmt.Errorf("can't get absolute path of '%s': %s", appsecRuleSource, err) } - waapRuleFilename := filepath.Base(waapRuleSource) + appsecRuleFilename := filepath.Base(appsecRuleSource) - // runtime/hub/waap-rules/author/waap-rule - hubDirWaapRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubWaapRule.RemotePath)) + // runtime/hub/appsec-rules/author/appsec-rule + hubDirAppsecRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubAppsecRule.RemotePath)) - // runtime/waap-rules/ - waapRuleDirDest := fmt.Sprintf("%s/waap-rules/", t.RuntimePath) + // runtime/appsec-rules/ + appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath) - if err := os.MkdirAll(hubDirWaapRuleDest, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %s", hubDirWaapRuleDest, err) + if err := os.MkdirAll(hubDirAppsecRuleDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", hubDirAppsecRuleDest, err) } - if err := os.MkdirAll(waapRuleDirDest, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %s", waapRuleDirDest, err) + if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err) } - // runtime/hub/waap-rules/crowdsecurity/rule.yaml - hubDirWaapRulePath := filepath.Join(waapRuleDirDest, waapRuleFilename) - if err := Copy(waapRuleSource, hubDirWaapRulePath); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %s", waapRuleSource, hubDirWaapRulePath, err) + // runtime/hub/appsec-rules/crowdsecurity/rule.yaml + hubDirAppsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename) + if err := Copy(appsecRuleSource, hubDirAppsecRulePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %s", appsecRuleSource, hubDirAppsecRulePath, err) } - // runtime/waap-rules/rule.yaml - waapRulePath := filepath.Join(waapRuleDirDest, waapRuleFilename) - if err := os.Symlink(hubDirWaapRulePath, waapRulePath); err != nil { + // runtime/appsec-rules/rule.yaml + appsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename) + if err := os.Symlink(hubDirAppsecRulePath, appsecRulePath); err != nil { if !os.IsExist(err) { - return fmt.Errorf("unable to symlink waap-rule '%s' to '%s': %s", hubDirWaapRulePath, waapRulePath, err) + return fmt.Errorf("unable to symlink appsec-rule '%s' to '%s': %s", hubDirAppsecRulePath, appsecRulePath, err) } } } else { - customWaapRuleExist := false + customAppsecRuleExist := false for _, customPath := range t.CustomItemsLocation { - // we check if its a custom waap-rule - customWaapRulePath := filepath.Join(customPath, waaprule) - if _, err := os.Stat(customWaapRulePath); os.IsNotExist(err) { + // we check if its a custom appsec-rule + customAppsecRulePath := filepath.Join(customPath, appsecrule) + if _, err := os.Stat(customAppsecRulePath); os.IsNotExist(err) { continue } - customWaapRulePathSplit := strings.Split(customWaapRulePath, "/") - customWappRuleName := customWaapRulePathSplit[len(customWaapRulePathSplit)-1] + customAppsecRulePathSplit := strings.Split(customAppsecRulePath, "/") + customAppsecRuleName := customAppsecRulePathSplit[len(customAppsecRulePathSplit)-1] - waapRuleDirDest := fmt.Sprintf("%s/waap-rules/", t.RuntimePath) - if err := os.MkdirAll(waapRuleDirDest, os.ModePerm); err != nil { - return fmt.Errorf("unable to create folder '%s': %s", waapRuleDirDest, err) + appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath) + if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err) } - // runtime/waap-rules/ - customWaapRuleDest := fmt.Sprintf("%s/waap-rules/%s", t.RuntimePath, customWappRuleName) + // runtime/appsec-rules/ + customAppsecRuleDest := fmt.Sprintf("%s/appsec-rules/%s", t.RuntimePath, customAppsecRuleName) // if path to postoverflow exist, copy it - if err := Copy(customWaapRulePath, customWaapRuleDest); err != nil { + if err := Copy(customAppsecRulePath, customAppsecRuleDest); err != nil { continue } - customWaapRuleExist = true + customAppsecRuleExist = true break } - if !customWaapRuleExist { - return fmt.Errorf("couldn't find custom waap-rule '%s' in the following location: %+v", waaprule, t.CustomItemsLocation) + if !customAppsecRuleExist { + return fmt.Errorf("couldn't find custom appsec-rule '%s' in the following location: %+v", appsecrule, t.CustomItemsLocation) } } } @@ -562,12 +562,12 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } //hardcode bouncer key - cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "waaptests", "-k", TestBouncerApiKey} + cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey} cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) output, err = cscliBouncerCmd.CombinedOutput() if err != nil { - if !strings.Contains(string(output), "unable to create bouncer: bouncer waaptests already exists") { + if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") { fmt.Println(string(output)) return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) } @@ -579,9 +579,9 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { crowdsecDaemon.Start() - //wait for the waap port to be available - if _, err := IsAlive(DefaultWaapHost); err != nil { - return fmt.Errorf("waap is down: %s", err) + //wait for the appsec port to be available + if _, err := IsAlive(DefaultAppsecHost); err != nil { + return fmt.Errorf("appsec is down: %s", err) } // check if the target is available @@ -608,17 +608,17 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { if t.Config.ExpectedNucleiFailure { if err != nil && errors.Is(err, NucleiTemplateFail) { - log.Infof("WAAP test %s failed as expected", t.Name) + log.Infof("Appsec test %s failed as expected", t.Name) t.Success = true } else { - log.Errorf("WAAP test %s failed: %s", t.Name, err) + log.Errorf("Appsec test %s failed: %s", t.Name, err) } } else { if err == nil { - log.Infof("WAAP test %s succeeded", t.Name) + log.Infof("Appsec test %s succeeded", t.Name) t.Success = true } else { - log.Errorf("WAAP test %s failed: %s", t.Name, err) + log.Errorf("Appsec test %s failed: %s", t.Name, err) } } crowdsecDaemon.Process.Kill() @@ -875,23 +875,23 @@ func (t *HubTestItem) Run() error { return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) } - // create the waap-configs dir - if err = os.MkdirAll(filepath.Join(t.RuntimePath, "waap-configs"), os.ModePerm); err != nil { + // create the appsec-configs dir + if err = os.MkdirAll(filepath.Join(t.RuntimePath, "appsec-configs"), os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) } - //if it's a waap rule test, we need acquis and waap profile - if len(t.Config.WaapRules) > 0 { + //if it's an appsec rule test, we need acquis and appsec profile + if len(t.Config.AppsecRules) > 0 { // copy template acquis file to runtime folder log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath) if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err) } - log.Infof("copying %s to %s", t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml")) - // copy template waap-config file to runtime folder - if err = Copy(t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml")); err != nil { - return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateWaapProfilePath, filepath.Join(t.RuntimePath, "waap-configs", "config.yaml"), err) + log.Infof("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")) + // copy template appsec-config file to runtime folder + if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err) } } else { //otherwise we drop a blank acquis file if err = os.WriteFile(t.RuntimeAcquisFilePath, []byte(""), os.ModePerm); err != nil { diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index b264b787064..5f816fbe6fc 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -116,7 +116,7 @@ func IsAlive(target string) (bool, error) { for { conn, err := net.Dial("tcp", target) if err == nil { - log.Debugf("waap is up after %s", time.Since(start)) + log.Debugf("appsec is up after %s", time.Since(start)) conn.Close() return true, nil } diff --git a/pkg/waf/loader.go b/pkg/waf/loader.go index f80f09f1266..e58c1650f84 100644 --- a/pkg/waf/loader.go +++ b/pkg/waf/loader.go @@ -16,7 +16,7 @@ func LoadAppsecRules(hubInstance *cwhub.Hub) error { hub = hubInstance - for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.WAAP_RULES) { + for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.APPSEC_RULES) { //log.Infof("loading %s", hubWafRuleItem.LocalPath) if !hubAppsecRuleItem.State.Installed { continue @@ -38,8 +38,8 @@ func LoadAppsecRules(hubInstance *cwhub.Hub) error { continue } - if rule.Type != WAAP_RULE { - log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, WAAP_RULE, hubAppsecRuleItem.State.LocalPath) + if rule.Type != cwhub.APPSEC_RULES { + log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, cwhub.APPSEC_RULES, hubAppsecRuleItem.State.LocalPath) continue } diff --git a/pkg/waf/request.go b/pkg/waf/request.go index 66b72467cd1..b0fe756767d 100644 --- a/pkg/waf/request.go +++ b/pkg/waf/request.go @@ -15,11 +15,11 @@ import ( ) const ( - URIHeaderName = "X-Crowdsec-Waap-Uri" - VerbHeaderName = "X-Crowdsec-Waap-Verb" - HostHeaderName = "X-Crowdsec-Waap-Host" - IPHeaderName = "X-Crowdsec-Waap-Ip" - APIKeyHeaderName = "X-Crowdsec-Waap-Api-Key" + URIHeaderName = "X-Crowdsec-Appsec-Uri" + VerbHeaderName = "X-Crowdsec-Appsec-Verb" + HostHeaderName = "X-Crowdsec-Appsec-Host" + IPHeaderName = "X-Crowdsec-Appsec-Ip" + APIKeyHeaderName = "X-Crowdsec-Appsec-Api-Key" ) type ParsedRequest struct { @@ -40,7 +40,7 @@ type ParsedRequest struct { ResponseChannel chan AppsecTempResponse `json:"-"` IsInBand bool `json:"-"` IsOutBand bool `json:"-"` - WaapEngine string `json:"waap_engine,omitempty"` + AppsecEngine string `json:"appsec_engine,omitempty"` RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` } @@ -313,12 +313,12 @@ func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { remoteAddrNormalized := "" host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - log.Errorf("Invalid waap remote IP source %v: %s", r.RemoteAddr, err.Error()) + log.Errorf("Invalid appsec remote IP source %v: %s", r.RemoteAddr, err.Error()) remoteAddrNormalized = r.RemoteAddr } else { ip := net.ParseIP(host) if ip == nil { - log.Errorf("Invalid waap remote IP address source %v: %s", r.RemoteAddr, err.Error()) + log.Errorf("Invalid appsec remote IP address source %v: %s", r.RemoteAddr, err.Error()) remoteAddrNormalized = r.RemoteAddr } else { remoteAddrNormalized = ip.String() diff --git a/pkg/waf/waap.go b/pkg/waf/waap.go index 34ebc079fcb..546040a8599 100644 --- a/pkg/waf/waap.go +++ b/pkg/waf/waap.go @@ -184,7 +184,7 @@ func (wc *AppsecConfig) LoadByPath(file string) error { } func (wc *AppsecConfig) Load(configName string) error { - appsecConfigs := hub.GetItemMap(cwhub.WAAP_CONFIGS) + appsecConfigs := hub.GetItemMap(cwhub.APPSEC_CONFIGS) for _, hubAppsecConfigItem := range appsecConfigs { if !hubAppsecConfigItem.State.Installed { diff --git a/pkg/waf/waap_rules_collection.go b/pkg/waf/waap_rules_collection.go index fdb96ba75cb..820642d686d 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/waf/waap_rules_collection.go @@ -16,7 +16,7 @@ type AppsecCollection struct { Rules []string } -var WAAP_RULE = "waap-rule" +var APPSEC_RULE = "appsec-rule" // to be filled w/ seb update type AppsecCollectionConfig struct { From 059c0adb93ad277e69d803f1561bab3b12769e2f Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 22:49:11 +0100 Subject: [PATCH 238/263] appsec renaming, part 5 --- cmd/crowdsec-cli/appsec_rules.go | 8 +++--- cmd/crowdsec/crowdsec.go | 4 +-- pkg/acquisition/acquisition.go | 2 +- pkg/acquisition/modules/appsec/appsec.go | 24 ++++++++-------- .../modules/appsec/appsec_runner.go | 28 +++++++++---------- pkg/acquisition/modules/appsec/utils.go | 20 ++++++------- .../appsec_rule}/modsec_rule_test.go | 2 +- .../appsec_rule}/modsecurity.go | 20 ++++++------- .../waap_rule => appsec/appsec_rule}/types.go | 2 +- .../appsec_rule}/waap_rule.go | 6 ++-- pkg/{waf => appsec}/coraza_logger.go | 2 +- pkg/{waf => appsec}/loader.go | 3 +- pkg/{waf => appsec}/request.go | 4 +-- pkg/{waf => appsec}/request_test.go | 2 +- pkg/{waf => appsec}/tx.go | 2 +- pkg/{waf => appsec}/waap.go | 2 +- pkg/{waf => appsec}/waap_rules_collection.go | 22 +++++++-------- pkg/{waf => appsec}/waf_helpers.go | 2 +- pkg/csconfig/crowdsec_service.go | 2 -- pkg/hubtest/coverage.go | 4 +-- 20 files changed, 78 insertions(+), 83 deletions(-) rename pkg/{waf/waap_rule => appsec/appsec_rule}/modsec_rule_test.go (99%) rename pkg/{waf/waap_rule => appsec/appsec_rule}/modsecurity.go (82%) rename pkg/{waf/waap_rule => appsec/appsec_rule}/types.go (85%) rename pkg/{waf/waap_rule => appsec/appsec_rule}/waap_rule.go (88%) rename pkg/{waf => appsec}/coraza_logger.go (99%) rename pkg/{waf => appsec}/loader.go (95%) rename pkg/{waf => appsec}/request.go (99%) rename pkg/{waf => appsec}/request_test.go (99%) rename pkg/{waf => appsec}/tx.go (99%) rename pkg/{waf => appsec}/waap.go (99%) rename pkg/{waf => appsec}/waap_rules_collection.go (81%) rename pkg/{waf => appsec}/waf_helpers.go (99%) diff --git a/cmd/crowdsec-cli/appsec_rules.go b/cmd/crowdsec-cli/appsec_rules.go index a5da34c3c4c..9fa2f85d6e9 100644 --- a/cmd/crowdsec-cli/appsec_rules.go +++ b/cmd/crowdsec-cli/appsec_rules.go @@ -11,9 +11,9 @@ import ( "gopkg.in/yaml.v3" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" ) func NewAppsecRulesCmd() *cobra.Command { @@ -127,7 +127,7 @@ func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, arg hub, _ := require.Hub(csConfig, nil) for _, name := range args { hubItem := hub.GetItem(itemType.name, name) - appsecRule := waf.AppsecCollectionConfig{} + appsecRule := appsec.AppsecCollectionConfig{} yamlContent, err := os.ReadFile(hubItem.State.LocalPath) if err != nil { return fmt.Errorf("unable to read file %s : %s", hubItem.State.LocalPath, err) @@ -136,7 +136,7 @@ func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, arg return fmt.Errorf("unable to unmarshal yaml file %s : %s", hubItem.State.LocalPath, err) } - for _, ruleType := range waap_rule.SupportedTypes() { + for _, ruleType := range appsec_rule.SupportedTypes() { fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) for _, rule := range appsecRule.Rules { convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 64c4797279b..1e0d54c0736 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -13,12 +13,12 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition" + "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/waf" ) func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { @@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er return nil, fmt.Errorf("while loading scenarios: %w", err) } - if err := waf.LoadAppsecRules(hub); err != nil { + if err := appsec.LoadAppsecRules(hub); err != nil { return nil, fmt.Errorf("while loading appsec rules: %w", err) } diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 960de824144..33602936369 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -77,7 +77,7 @@ var AcquisitionSources = map[string]func() DataSource{ "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "loki": func() DataSource { return &lokiacquisition.LokiSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, - "waf": func() DataSource { return &appsecacquisition.AppsecSource{} }, + "appsec": func() DataSource { return &appsecacquisition.AppsecSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index e128ba6aec5..d55c077fb78 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -11,8 +11,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/trace" "github.com/google/uuid" "github.com/pkg/errors" @@ -52,9 +52,9 @@ type AppsecSource struct { server *http.Server addr string outChan chan types.Event - InChan chan waf.ParsedRequest - AppsecRuntime *waf.AppsecRuntimeConfig - AppsecConfigs map[string]waf.AppsecConfig + InChan chan appsec.ParsedRequest + AppsecRuntime *appsec.AppsecRuntimeConfig + AppsecConfigs map[string]appsec.AppsecConfig lapiURL string AuthCache AuthCache AppsecRunners []AppsecRunner //one for each go-routine @@ -119,7 +119,7 @@ func (wc *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { wc.config.Mode = configuration.TAIL_MODE } - // always have at least one waf routine + // always have at least one appsec routine if wc.config.Routines == 0 { wc.config.Routines = 1 } @@ -150,12 +150,12 @@ func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { err := w.UnmarshalConfig(yamlConfig) if err != nil { - return errors.Wrap(err, "unable to parse waf configuration") + return errors.Wrap(err, "unable to parse appsec configuration") } w.logger = logger w.logger.Logger.SetLevel(*w.config.LogLevel) - w.logger.Tracef("WAF configuration: %+v", w.config) + w.logger.Tracef("Appsec configuration: %+v", w.config) if w.config.AuthCacheDuration == nil { w.config.AuthCacheDuration = &DefaultAuthCacheDuration @@ -169,8 +169,8 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { Handler: w.mux, } - w.InChan = make(chan waf.ParsedRequest) - appsecCfg := waf.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} + w.InChan = make(chan appsec.ParsedRequest) + appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} //let's load the associated appsec_config: if w.config.AppsecConfigPath != "" { @@ -317,8 +317,8 @@ func (w *AppsecSource) IsAuth(apiKey string) bool { // should this be in the runner ? func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { - apiKey := r.Header.Get(waf.APIKeyHeaderName) - clientIP := r.Header.Get(waf.IPHeaderName) + apiKey := r.Header.Get(appsec.APIKeyHeaderName) + clientIP := r.Header.Get(appsec.IPHeaderName) remoteIP := r.RemoteAddr if apiKey == "" { w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) @@ -339,7 +339,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { } // parse the request only once - parsedRequest, err := waf.NewParsedRequestFromRequest(r) + parsedRequest, err := appsec.NewParsedRequestFromRequest(r) if err != nil { log.Errorf("%s", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 6ce7e57df40..b2b10ea6b81 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -8,8 +8,8 @@ import ( "github.com/crowdsecurity/coraza/v3" corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -18,9 +18,9 @@ import ( // that's the runtime structure of the Application security engine as seen from the acquis type AppsecRunner struct { outChan chan types.Event - inChan chan waf.ParsedRequest + inChan chan appsec.ParsedRequest UUID string - AppsecRuntime *waf.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc. + AppsecRuntime *appsec.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc. AppsecInbandEngine coraza.WAF AppsecOutbandEngine coraza.WAF logger *log.Entry @@ -44,7 +44,7 @@ func (r *AppsecRunner) Init(datadir string) error { outBandLogger := r.logger.Dup().WithField("band", "outband") //setting up inband engine - inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(inBandLogger)) + inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(inBandLogger)) if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection { inbandCfg = inbandCfg.WithRequestBodyAccess() } else { @@ -59,7 +59,7 @@ func (r *AppsecRunner) Init(datadir string) error { } //setting up outband engine - outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(waf.NewCrzLogger(outBandLogger)) + outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(outBandLogger)) if !r.AppsecRuntime.Config.OutOfBandOptions.DisableBodyInspection { outbandCfg = outbandCfg.WithRequestBodyAccess() } else { @@ -101,7 +101,7 @@ func (r *AppsecRunner) Init(datadir string) error { return nil } -func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.ParsedRequest) error { +func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *appsec.ParsedRequest) error { var in *corazatypes.Interruption var err error request.Tx = tx @@ -185,21 +185,21 @@ func (r *AppsecRunner) processRequest(tx waf.ExtendedTransaction, request *waf.P return nil } -func (r *AppsecRunner) ProcessInBandRules(request *waf.ParsedRequest) error { - tx := waf.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) +func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { + tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) r.AppsecRuntime.InBandTx = tx err := r.processRequest(tx, request) return err } -func (r *AppsecRunner) ProcessOutOfBandRules(request *waf.ParsedRequest) error { - tx := waf.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) +func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { + tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) r.AppsecRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) return err } -func (r *AppsecRunner) handleInBandInterrupt(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { //create the associated event for crowdsec itself evt, err := EventFromRequest(request) if err != nil { @@ -248,7 +248,7 @@ func (r *AppsecRunner) handleInBandInterrupt(request *waf.ParsedRequest) { } } -func (r *AppsecRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { evt, err := EventFromRequest(request) if err != nil { //let's not interrupt the pipeline for this @@ -284,7 +284,7 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *waf.ParsedRequest) { } } -func (r *AppsecRunner) handleRequest(request *waf.ParsedRequest) { +func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { r.logger.Debugf("Requests handled by runner %s", request.UUID) r.AppsecRuntime.ClearResponse() @@ -334,7 +334,7 @@ func (r *AppsecRunner) Run(t *tomb.Tomb) error { for { select { case <-t.Dying(): - r.logger.Infof("Waf Runner is dying") + r.logger.Infof("Appsec Runner is dying") return nil case request := <-r.inChan: r.handleRequest(&request) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index a228cf72d1a..008811ae782 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -7,9 +7,9 @@ import ( "github.com/crowdsecurity/coraza/v3/collection" "github.com/crowdsecurity/coraza/v3/types/variables" + "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -51,7 +51,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.Meta = append(alert.Meta, &meta) } alert.EventsCount = ptr.Of(int32(1)) - alert.Labels = []string{"waf"} //don't know what to do about this + alert.Labels = []string{"appsec"} //don't know what to do about this alert.Leakspeed = ptr.Of("") msg := fmt.Sprintf("Application Security Engine alert: %s", inEvt.Waap.MatchedRules.GetName()) alert.Message = &msg @@ -68,7 +68,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { return &evt, nil } -func EventFromRequest(r *waf.ParsedRequest) (types.Event, error) { +func EventFromRequest(r *appsec.ParsedRequest) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -91,11 +91,11 @@ func EventFromRequest(r *waf.ParsedRequest) (types.Event, error) { evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "crowdsec-waap"}, //FIXME: use the labels from the acquis + Labels: map[string]string{"type": "crowdsec-appsec"}, //FIXME: use the labels from the acquis Process: true, Module: "appsec", Src: "appsec", - Raw: "dummy-waap-data", //we discard empty Line.Raw items :) + Raw: "dummy-appsec-data", //we discard empty Line.Raw items :) } evt.Waap = types.AppsecEvent{} @@ -110,19 +110,19 @@ func LogAppsecEvent(evt *types.Event, logger *log.Entry) { if evt.Meta["waap_interrupted"] == "true" { logger.WithFields(log.Fields{ - "module": "waf", + "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) } else if evt.Parsed["outofband_interrupted"] == "true" { logger.WithFields(log.Fields{ - "module": "waf", + "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, }).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) } else { logger.WithFields(log.Fields{ - "module": "waf", + "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, }).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) @@ -130,7 +130,7 @@ func LogAppsecEvent(evt *types.Event, logger *log.Entry) { } -func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequest) error { +func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedRequest) error { if evt == nil { //an error was already emitted, let's not spam the logs @@ -206,7 +206,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *waf.ParsedRequ hash := "NOT_SET" ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID()) - if details, ok := waf.AppsecRulesDetails[rule.Rule().ID()]; ok { + if details, ok := appsec.AppsecRulesDetails[rule.Rule().ID()]; ok { //Only set them for custom rules, not for rules written in seclang name = details.Name version = details.Version diff --git a/pkg/waf/waap_rule/modsec_rule_test.go b/pkg/appsec/appsec_rule/modsec_rule_test.go similarity index 99% rename from pkg/waf/waap_rule/modsec_rule_test.go rename to pkg/appsec/appsec_rule/modsec_rule_test.go index 2232540ec33..d919dce25f3 100644 --- a/pkg/waf/waap_rule/modsec_rule_test.go +++ b/pkg/appsec/appsec_rule/modsec_rule_test.go @@ -1,4 +1,4 @@ -package waap_rule +package appsec_rule import "testing" diff --git a/pkg/waf/waap_rule/modsecurity.go b/pkg/appsec/appsec_rule/modsecurity.go similarity index 82% rename from pkg/waf/waap_rule/modsecurity.go rename to pkg/appsec/appsec_rule/modsecurity.go index 1ac7486dbe2..760c697ccaf 100644 --- a/pkg/waf/waap_rule/modsecurity.go +++ b/pkg/appsec/appsec_rule/modsecurity.go @@ -1,4 +1,4 @@ -package waap_rule +package appsec_rule import ( "fmt" @@ -50,9 +50,9 @@ var bodyTypeMatch map[string]string = map[string]string{ "urlencoded": "URLENCODED", } -func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, []uint32, error) { +func (m *ModsecurityRule) Build(rule *CustomRule, appsecRuleName string) (string, []uint32, error) { - rules, err := m.buildRules(rule, waapRuleName, false, 0, 0) + rules, err := m.buildRules(rule, appsecRuleName, false, 0, 0) if err != nil { return "", nil, err @@ -62,9 +62,9 @@ func (m *ModsecurityRule) Build(rule *CustomRule, waapRuleName string) (string, return strings.Join(rules, "\n"), m.ids, nil } -func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string, depth int) uint32 { +func (m *ModsecurityRule) generateRuleID(rule *CustomRule, appsecRuleName string, depth int) uint32 { h := fnv.New32a() - h.Write([]byte(waapRuleName)) + h.Write([]byte(appsecRuleName)) h.Write([]byte(rule.Match.Type)) h.Write([]byte(rule.Match.Value)) h.Write([]byte(fmt.Sprintf("%d", depth))) @@ -79,7 +79,7 @@ func (m *ModsecurityRule) generateRuleID(rule *CustomRule, waapRuleName string, return id } -func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and bool, toSkip int, depth int) ([]string, error) { +func (m *ModsecurityRule) buildRules(rule *CustomRule, appsecRuleName string, and bool, toSkip int, depth int) ([]string, error) { ret := make([]string, 0) if len(rule.And) != 0 && len(rule.Or) != 0 { @@ -89,9 +89,8 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and if rule.And != nil { for c, andRule := range rule.And { depth++ - //subName := fmt.Sprintf("%s_and_%d", waapRuleName, c) lastRule := c == len(rule.And)-1 // || len(rule.Or) == 0 - rules, err := m.buildRules(&andRule, waapRuleName, !lastRule, 0, depth) + rules, err := m.buildRules(&andRule, appsecRuleName, !lastRule, 0, depth) if err != nil { return nil, err } @@ -102,9 +101,8 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and if rule.Or != nil { for c, orRule := range rule.Or { depth++ - //subName := fmt.Sprintf("%s_or_%d", waapRuleName, c) skip := len(rule.Or) - c - 1 - rules, err := m.buildRules(&orRule, waapRuleName, false, skip, depth) + rules, err := m.buildRules(&orRule, appsecRuleName, false, skip, depth) if err != nil { return nil, err } @@ -147,7 +145,7 @@ func (m *ModsecurityRule) buildRules(rule *CustomRule, waapRuleName string, and } //Should phase:2 be configurable? - r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s',tag:'crowdsec-%s'`, m.generateRuleID(rule, waapRuleName, depth), waapRuleName, waapRuleName)) + r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s',tag:'crowdsec-%s'`, m.generateRuleID(rule, appsecRuleName, depth), appsecRuleName, appsecRuleName)) if rule.Transform != nil { for _, transform := range rule.Transform { diff --git a/pkg/waf/waap_rule/types.go b/pkg/appsec/appsec_rule/types.go similarity index 85% rename from pkg/waf/waap_rule/types.go rename to pkg/appsec/appsec_rule/types.go index ba7bfdafbd6..13716975a05 100644 --- a/pkg/waf/waap_rule/types.go +++ b/pkg/appsec/appsec_rule/types.go @@ -1,4 +1,4 @@ -package waap_rule +package appsec_rule const ( ModsecurityRuleType = "modsecurity" diff --git a/pkg/waf/waap_rule/waap_rule.go b/pkg/appsec/appsec_rule/waap_rule.go similarity index 88% rename from pkg/waf/waap_rule/waap_rule.go rename to pkg/appsec/appsec_rule/waap_rule.go index 00e6cefd113..c011e58fb02 100644 --- a/pkg/waf/waap_rule/waap_rule.go +++ b/pkg/appsec/appsec_rule/waap_rule.go @@ -1,4 +1,4 @@ -package waap_rule +package appsec_rule import ( "fmt" @@ -43,7 +43,7 @@ type CustomRule struct { BodyType string `yaml:"body_type,omitempty"` } -func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, []uint32, error) { +func (v *CustomRule) Convert(ruleType string, appsecRuleName string) (string, []uint32, error) { if v.Zones == nil && v.And == nil && v.Or == nil { return "", nil, fmt.Errorf("no zones defined") @@ -60,7 +60,7 @@ func (v *CustomRule) Convert(ruleType string, waapRuleName string) (string, []ui switch ruleType { case ModsecurityRuleType: r := ModsecurityRule{} - return r.Build(v, waapRuleName) + return r.Build(v, appsecRuleName) default: return "", nil, fmt.Errorf("unknown rule format '%s'", ruleType) } diff --git a/pkg/waf/coraza_logger.go b/pkg/appsec/coraza_logger.go similarity index 99% rename from pkg/waf/coraza_logger.go rename to pkg/appsec/coraza_logger.go index dfeeb669e47..372a0098ecc 100644 --- a/pkg/waf/coraza_logger.go +++ b/pkg/appsec/coraza_logger.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "fmt" diff --git a/pkg/waf/loader.go b/pkg/appsec/loader.go similarity index 95% rename from pkg/waf/loader.go rename to pkg/appsec/loader.go index e58c1650f84..43bd7d78b1e 100644 --- a/pkg/waf/loader.go +++ b/pkg/appsec/loader.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "os" @@ -17,7 +17,6 @@ func LoadAppsecRules(hubInstance *cwhub.Hub) error { hub = hubInstance for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.APPSEC_RULES) { - //log.Infof("loading %s", hubWafRuleItem.LocalPath) if !hubAppsecRuleItem.State.Installed { continue } diff --git a/pkg/waf/request.go b/pkg/appsec/request.go similarity index 99% rename from pkg/waf/request.go rename to pkg/appsec/request.go index b0fe756767d..9979caf9006 100644 --- a/pkg/waf/request.go +++ b/pkg/appsec/request.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "encoding/json" @@ -266,7 +266,7 @@ func (r *ReqDumpFilter) ToJSON() error { return nil } -// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the Waap Engine +// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { var err error body := make([]byte, 0) diff --git a/pkg/waf/request_test.go b/pkg/appsec/request_test.go similarity index 99% rename from pkg/waf/request_test.go rename to pkg/appsec/request_test.go index 2625e11f5d3..b05ecbde62b 100644 --- a/pkg/waf/request_test.go +++ b/pkg/appsec/request_test.go @@ -1,4 +1,4 @@ -package waf +package appsec import "testing" diff --git a/pkg/waf/tx.go b/pkg/appsec/tx.go similarity index 99% rename from pkg/waf/tx.go rename to pkg/appsec/tx.go index 1e2c83331d9..47da19d1556 100644 --- a/pkg/waf/tx.go +++ b/pkg/appsec/tx.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "github.com/crowdsecurity/coraza/v3" diff --git a/pkg/waf/waap.go b/pkg/appsec/waap.go similarity index 99% rename from pkg/waf/waap.go rename to pkg/appsec/waap.go index 546040a8599..0477642dd5b 100644 --- a/pkg/waf/waap.go +++ b/pkg/appsec/waap.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "fmt" diff --git a/pkg/waf/waap_rules_collection.go b/pkg/appsec/waap_rules_collection.go similarity index 81% rename from pkg/waf/waap_rules_collection.go rename to pkg/appsec/waap_rules_collection.go index 820642d686d..6a404e9c7e4 100644 --- a/pkg/waf/waap_rules_collection.go +++ b/pkg/appsec/waap_rules_collection.go @@ -1,12 +1,12 @@ -package waf +package appsec import ( "os" "path/filepath" "strings" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" log "github.com/sirupsen/logrus" ) @@ -20,13 +20,13 @@ var APPSEC_RULE = "appsec-rule" // to be filled w/ seb update type AppsecCollectionConfig struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - Debug bool `yaml:"debug"` - Description string `yaml:"description"` - SecLangFilesRules []string `yaml:"seclang_files_rules"` - SecLangRules []string `yaml:"seclang_rules"` - Rules []waap_rule.CustomRule `yaml:"rules"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Debug bool `yaml:"debug"` + Description string `yaml:"description"` + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + Rules []appsec_rule.CustomRule `yaml:"rules"` Labels map[string]interface{} `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow @@ -47,7 +47,7 @@ type RulesDetails struct { var AppsecRulesDetails = make(map[int]RulesDetails) func LoadCollection(pattern string) ([]AppsecCollection, error) { - //FIXME: have a proper logger here, inheriting from waap-config to have consistent log levels + //FIXME: have a proper logger here, inheriting from appsec-config to have consistent log levels ret := make([]AppsecCollection, 0) for _, appsecRule := range appsecRules { @@ -100,7 +100,7 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { if appsecRule.Rules != nil { for _, rule := range appsecRule.Rules { - strRule, rulesId, err := rule.Convert(waap_rule.ModsecurityRuleType, appsecRule.Name) + strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name) if err != nil { log.Errorf("unable to convert rule %s : %s", rule.Name, err) return nil, err diff --git a/pkg/waf/waf_helpers.go b/pkg/appsec/waf_helpers.go similarity index 99% rename from pkg/waf/waf_helpers.go rename to pkg/appsec/waf_helpers.go index fa43c88482a..605d42ccb56 100644 --- a/pkg/waf/waf_helpers.go +++ b/pkg/appsec/waf_helpers.go @@ -1,4 +1,4 @@ -package waf +package appsec import ( "github.com/crowdsecurity/crowdsec/pkg/types" diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 77a3ed63078..dc226cfd6ac 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -11,8 +11,6 @@ import ( "github.com/crowdsecurity/go-cs-lib/ptr" ) -var DataDir string // FIXME: find a better way to pass this to the waf - // CrowdsecServiceCfg contains the location of parsers/scenarios/... and acquisition files type CrowdsecServiceCfg struct { Enable *bool `yaml:"enable"` diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index ce4083573da..edbe10454bf 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -7,8 +7,8 @@ import ( "path/filepath" "strings" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/waf/waap_rule" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -55,7 +55,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { } for _, appsecRulesFile := range configFileData.AppsecRules { - appsecRuleData := &waap_rule.CustomRule{} + appsecRuleData := &appsec_rule.CustomRule{} yamlFile, err := os.ReadFile(appsecRulesFile) if err != nil { log.Printf("unable to open appsec rule '%s': %s", appsecRulesFile, err) From 722ce46946581e1113ae375e51616f37f5217108 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Mon, 4 Dec 2023 23:48:48 +0100 Subject: [PATCH 239/263] remove useless check --- pkg/appsec/loader.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/appsec/loader.go b/pkg/appsec/loader.go index 43bd7d78b1e..fa13cb03ddd 100644 --- a/pkg/appsec/loader.go +++ b/pkg/appsec/loader.go @@ -37,11 +37,6 @@ func LoadAppsecRules(hubInstance *cwhub.Hub) error { continue } - if rule.Type != cwhub.APPSEC_RULES { - log.Warnf("unexpected type %s instead of %s for file %s", rule.Type, cwhub.APPSEC_RULES, hubAppsecRuleItem.State.LocalPath) - continue - } - rule.hash = hubAppsecRuleItem.State.LocalHash rule.version = hubAppsecRuleItem.Version From 1a1f4f61693731804f71201925997161a044ef4c Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 00:15:29 +0100 Subject: [PATCH 240/263] do not spam with "unknown" metrics --- cmd/crowdsec-cli/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 20cbed6dfe2..5b24dc84c91 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -249,7 +249,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error } appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival default: - log.Infof("unknown: %+v", fam.Name) + log.Debugf("unknown: %+v", fam.Name) continue } } From 52c1e16216d99006b8330ce9a27e12b9032536a1 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 01:00:59 +0100 Subject: [PATCH 241/263] more debug when loading rules --- pkg/acquisition/modules/appsec/appsec_runner.go | 4 ++++ pkg/appsec/waap.go | 5 +++-- pkg/appsec/waap_rules_collection.go | 17 +++++++++-------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index b2b10ea6b81..0887c725f2c 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -94,6 +94,9 @@ func (r *AppsecRunner) Init(datadir string) error { } } + r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules()) + r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules()) + if err != nil { return fmt.Errorf("unable to initialize outband engine : %w", err) } @@ -193,6 +196,7 @@ func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { } func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { + r.logger.Infof("Processing out of band rules") tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) r.AppsecRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) diff --git a/pkg/appsec/waap.go b/pkg/appsec/waap.go index 0477642dd5b..7f8cba53ecc 100644 --- a/pkg/appsec/waap.go +++ b/pkg/appsec/waap.go @@ -214,10 +214,11 @@ func (wc *AppsecConfig) Build() (*AppsecRuntimeConfig, error) { ret.Config = wc ret.DefaultRemediation = wc.DefaultRemediation + wc.Logger.Tracef("Loading config %+v", wc) //load rules for _, rule := range wc.OutOfBandRules { wc.Logger.Infof("loading outofband rule %s", rule) - collections, err := LoadCollection(rule) + collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader")) if err != nil { return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) } @@ -227,7 +228,7 @@ func (wc *AppsecConfig) Build() (*AppsecRuntimeConfig, error) { wc.Logger.Infof("Loaded %d outofband rules", len(ret.OutOfBandRules)) for _, rule := range wc.InBandRules { wc.Logger.Infof("loading inband rule %s", rule) - collections, err := LoadCollection(rule) + collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader")) if err != nil { return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) } diff --git a/pkg/appsec/waap_rules_collection.go b/pkg/appsec/waap_rules_collection.go index 6a404e9c7e4..d5da16f017c 100644 --- a/pkg/appsec/waap_rules_collection.go +++ b/pkg/appsec/waap_rules_collection.go @@ -46,8 +46,7 @@ type RulesDetails struct { // Is using the id is a good idea ? might be too specific to coraza and not easily reusable var AppsecRulesDetails = make(map[int]RulesDetails) -func LoadCollection(pattern string) ([]AppsecCollection, error) { - //FIXME: have a proper logger here, inheriting from appsec-config to have consistent log levels +func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, error) { ret := make([]AppsecCollection, 0) for _, appsecRule := range appsecRules { @@ -55,14 +54,14 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name) if err != nil { - log.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) + logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) continue } matched, ok := tmpMatch.(bool) if !ok { - log.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) + logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) continue } @@ -76,10 +75,11 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { if appsecRule.SecLangFilesRules != nil { for _, rulesFile := range appsecRule.SecLangFilesRules { + logger.Debugf("Adding rules from %s", rulesFile) fullPath := filepath.Join(hub.GetDataDir(), rulesFile) c, err := os.ReadFile(fullPath) if err != nil { - log.Errorf("unable to read file %s : %s", rulesFile, err) + logger.Errorf("unable to read file %s : %s", rulesFile, err) continue } for _, line := range strings.Split(string(c), "\n") { @@ -95,6 +95,7 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { } if appsecRule.SecLangRules != nil { + logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules) appsecCol.Rules = append(appsecCol.Rules, appsecRule.SecLangRules...) } @@ -102,10 +103,10 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { for _, rule := range appsecRule.Rules { strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name) if err != nil { - log.Errorf("unable to convert rule %s : %s", rule.Name, err) + logger.Errorf("unable to convert rule %s : %s", rule.Name, err) return nil, err } - log.Debugf("Adding rule %s", strRule) + logger.Debugf("Adding rule %s", strRule) appsecCol.Rules = append(appsecCol.Rules, strRule) //We only take the first id, as it's the one of the "main" rule @@ -117,7 +118,7 @@ func LoadCollection(pattern string) ([]AppsecCollection, error) { Name: appsecRule.Name, } } else { - log.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) + logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) } for _, id := range rulesId { From bb307dd339d98bbca7c6c97a5b30bdf7d2709c96 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 01:01:15 +0100 Subject: [PATCH 242/263] return an error if not appsec-rules matches --- pkg/appsec/waap_rules_collection.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/appsec/waap_rules_collection.go b/pkg/appsec/waap_rules_collection.go index d5da16f017c..4ccc639890f 100644 --- a/pkg/appsec/waap_rules_collection.go +++ b/pkg/appsec/waap_rules_collection.go @@ -1,6 +1,7 @@ package appsec import ( + "fmt" "os" "path/filepath" "strings" @@ -128,6 +129,9 @@ func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, erro } ret = append(ret, appsecCol) } + if len(ret) == 0 { + return nil, fmt.Errorf("no appsec-rules found for pattern %s", pattern) + } return ret, nil } From b86ac92b113e69cebd41320de14c1e676a70be4a Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 01:02:41 +0100 Subject: [PATCH 243/263] appsec renaming, part 6 --- pkg/appsec/{waap.go => appsec.go} | 0 pkg/appsec/appsec_rule/{waap_rule.go => appsec_rule.go} | 0 .../{waap_rules_collection.go => appsec_rules_collection.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename pkg/appsec/{waap.go => appsec.go} (100%) rename pkg/appsec/appsec_rule/{waap_rule.go => appsec_rule.go} (100%) rename pkg/appsec/{waap_rules_collection.go => appsec_rules_collection.go} (100%) diff --git a/pkg/appsec/waap.go b/pkg/appsec/appsec.go similarity index 100% rename from pkg/appsec/waap.go rename to pkg/appsec/appsec.go diff --git a/pkg/appsec/appsec_rule/waap_rule.go b/pkg/appsec/appsec_rule/appsec_rule.go similarity index 100% rename from pkg/appsec/appsec_rule/waap_rule.go rename to pkg/appsec/appsec_rule/appsec_rule.go diff --git a/pkg/appsec/waap_rules_collection.go b/pkg/appsec/appsec_rules_collection.go similarity index 100% rename from pkg/appsec/waap_rules_collection.go rename to pkg/appsec/appsec_rules_collection.go From cce83d1bdc0fc3355cefe2ae7f564ed5003cb52b Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 09:48:56 +0100 Subject: [PATCH 244/263] appsec renaming, part 7 --- pkg/acquisition/modules/appsec/README.md | 73 ------------------------ pkg/acquisition/modules/appsec/utils.go | 38 ++++++------ pkg/types/appsec_event.go | 59 ++++++++++--------- pkg/types/event.go | 2 +- 4 files changed, 49 insertions(+), 123 deletions(-) delete mode 100644 pkg/acquisition/modules/appsec/README.md diff --git a/pkg/acquisition/modules/appsec/README.md b/pkg/acquisition/modules/appsec/README.md deleted file mode 100644 index d2d4b23191f..00000000000 --- a/pkg/acquisition/modules/appsec/README.md +++ /dev/null @@ -1,73 +0,0 @@ -Ongoing poc for Coraza WAAP - -# Configuration pieces - -## Acquisition - -acquisition example: - -> `config/acquis.yaml` : - -```yaml -listen_addr: 127.0.0.1 -listen_port: 4241 -path: / -source: waf -labels: - type: waf -#routines: 1 -waap_config_path: config/waap-configs/mytest.yaml -``` - -## Waap config - -The waap config defines what rules that will be loaded by a given waap engine (associated with an acquis). - -> `config/waap-configs/mytest.yaml` - -```yaml -name: default -outofband_rules: -# - crowdsecurity/crs-waf -inband_rules: -# - crowdsecurity/crs-waf - - crowdsecurity/custom-waf-rule -default_remediation: ban -#on_load: -# - apply: -# - SetInBand() -pre_eval: - - filter: ClientIP != '127.0.0.1' - apply: - - SetAction("ban") -``` - -# Waap Rules - -For the above two to work, we need to have the two refered waap collection installed : `crowdsec/crs-default` and `crowdsec/vpatch-default`. You need to set hub_branch to ... - -```yaml -type: waap-rule -name: crowdsecurity/custom-waf-rule -seclang_rules: - - SecRule ARGS:ip ";" "t:none,phase:1,log,deny,msg:'semi colon test',id:2" - -#$_GET['bar'] matches [0-9]+ AND REQUEST_URI == "/joomla/index.php/component/users/" -#REQUEST_URI == /webui/create_user AND $_POST[username] == "cisco_tac_admin" -rules: - - target: ARGS - var: bar - match: "[0-9]+" - logic: AND - sub_rules: - - target: "REQUEST_URI" - match: /joomla/index.php/component/users/ - - target: "REQUEST_URI" - equals: /webui/create_user - logic: AND - sub_rules: - - target: ARGS_POST - var: username - equals: cisco_tac_admin -``` - diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 008811ae782..3776265395b 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -17,7 +17,7 @@ import ( func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { //if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI - if !inEvt.Waap.HasInBandMatches { + if !inEvt.Appsec.HasInBandMatches { return nil, nil } evt := types.Event{} @@ -53,11 +53,11 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"appsec"} //don't know what to do about this alert.Leakspeed = ptr.Of("") - msg := fmt.Sprintf("Application Security Engine alert: %s", inEvt.Waap.MatchedRules.GetName()) + msg := fmt.Sprintf("Application Security Engine alert: %s", inEvt.Appsec.MatchedRules.GetName()) alert.Message = &msg - alert.Scenario = ptr.Of(inEvt.Waap.MatchedRules.GetName()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() - alert.ScenarioHash = ptr.Of(inEvt.Waap.MatchedRules.GetHash()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetHash() - alert.ScenarioVersion = ptr.Of(inEvt.Waap.MatchedRules.GetVersion()) // @sbl : should we be able to do inEvt.Waap.MatchedRules.GetVersion() + alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName()) + alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash()) + alert.ScenarioVersion = ptr.Of(inEvt.Appsec.MatchedRules.GetVersion()) alert.Simulated = ptr.Of(false) alert.Source = &source alert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) @@ -81,7 +81,7 @@ func EventFromRequest(r *appsec.ParsedRequest) (types.Event, error) { "target_uri": r.URI, "method": r.Method, "req_uuid": r.Tx.ID(), - "source": "crowdsec-waap", + "source": "crowdsec-appsec", //TBD: //http_status @@ -97,7 +97,7 @@ func EventFromRequest(r *appsec.ParsedRequest) (types.Event, error) { Src: "appsec", Raw: "dummy-appsec-data", //we discard empty Line.Raw items :) } - evt.Waap = types.AppsecEvent{} + evt.Appsec = types.AppsecEvent{} return evt, nil } @@ -108,24 +108,24 @@ func LogAppsecEvent(evt *types.Event, logger *log.Entry) { req = req[:10] + ".." } - if evt.Meta["waap_interrupted"] == "true" { + if evt.Meta["appsec_interrupted"] == "true" { logger.WithFields(log.Fields{ "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, - }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) } else if evt.Parsed["outofband_interrupted"] == "true" { logger.WithFields(log.Fields{ "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, - }).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + }).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) } else { logger.WithFields(log.Fields{ "module": "appsec", "source": evt.Parsed["source_ip"], "target_uri": req, - }).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Waap.MatchedRules), evt.Waap.GetRuleIDs()) + }).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) } } @@ -151,8 +151,8 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR evt.Parsed = map[string]string{} } if req.IsInBand { - evt.Meta["waap_interrupted"] = "true" - evt.Meta["waap_action"] = req.Tx.Interruption().Action + evt.Meta["appsec_interrupted"] = "true" + evt.Meta["appsec_action"] = req.Tx.Interruption().Action evt.Parsed["inband_interrupted"] = "true" evt.Parsed["inband_action"] = req.Tx.Interruption().Action } else { @@ -160,8 +160,8 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR evt.Parsed["outofband_action"] = req.Tx.Interruption().Action } - if evt.Waap.Vars == nil { - evt.Waap.Vars = map[string]string{} + if evt.Appsec.Vars == nil { + evt.Appsec.Vars = map[string]string{} } req.Tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { @@ -178,7 +178,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR for _, collectionToKeep := range r.AppsecRuntime.CompiledVariablesTracking { match := collectionToKeep.MatchString(key) if match { - evt.Waap.Vars[key] = variable.Value() + evt.Appsec.Vars[key] = variable.Value() r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) } else { r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) @@ -196,9 +196,9 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR kind := "outofband" if req.IsInBand { kind = "inband" - evt.Waap.HasInBandMatches = true + evt.Appsec.HasInBandMatches = true } else { - evt.Waap.HasOutBandMatches = true + evt.Appsec.HasOutBandMatches = true } name := "NOT_SET" @@ -235,7 +235,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR "hash": hash, "version": version, } - evt.Waap.MatchedRules = append(evt.Waap.MatchedRules, corazaRule) + evt.Appsec.MatchedRules = append(evt.Appsec.MatchedRules, corazaRule) } return nil diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index 83ee06fd7e7..b31dfefefea 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -130,103 +130,102 @@ func (w MatchedRules) Kinds() []string { // filters func (w MatchedRules) ByID(id int) MatchedRules { - waap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { if rule["id"] == id { - waap = append(waap, rule) + ret = append(ret, rule) } } - return waap + return ret } func (w MatchedRules) ByKind(kind string) MatchedRules { - waap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { if rule["kind"] == kind { - waap = append(waap, rule) + ret = append(ret, rule) } } - return waap + return ret } func (w MatchedRules) ByTags(match []string) MatchedRules { - waap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { for _, tag := range rule["tags"].([]string) { for _, match_tag := range match { if tag == match_tag { - waap = append(waap, rule) + ret = append(ret, rule) break } } } } - return waap + return ret } func (w MatchedRules) ByTag(match string) MatchedRules { - waap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { for _, tag := range rule["tags"].([]string) { if tag == match { - waap = append(waap, rule) + ret = append(ret, rule) break } } } - return waap + return ret } func (w MatchedRules) ByTagRx(rx string) MatchedRules { - waap := MatchedRules{} + ret := MatchedRules{} re := regexp.MustCompile(rx) if re == nil { - return waap + return ret } for _, rule := range w { for _, tag := range rule["tags"].([]string) { - log.Infof("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) + log.Debugf("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) if re.MatchString(tag) { - waap = append(waap, rule) + ret = append(ret, rule) break } } } - return waap + return ret } func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules { - log.Infof("%s", w) - wap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { if rule["disruptive"] == is { - wap = append(wap, rule) + ret = append(ret, rule) } } - log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap)) + log.Debugf("ByDisruptiveness(%t) -> %d", is, len(ret)) - return wap + return ret } func (w MatchedRules) BySeverity(severity string) MatchedRules { - wap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { if rule["severity"] == severity { - wap = append(wap, rule) + ret = append(ret, rule) } } - log.Infof("BySeverity(%s) -> %d", severity, len(wap)) - return wap + log.Debugf("BySeverity(%s) -> %d", severity, len(ret)) + return ret } func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules { - wap := MatchedRules{} + ret := MatchedRules{} for _, rule := range w { if rule["accuracy"] == accuracy { - wap = append(wap, rule) + ret = append(ret, rule) } } - log.Infof("ByAccuracy(%s) -> %d", accuracy, len(wap)) - return wap + log.Debugf("ByAccuracy(%s) -> %d", accuracy, len(ret)) + return ret } diff --git a/pkg/types/event.go b/pkg/types/event.go index 2456df8672f..074241918d8 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -41,7 +41,7 @@ type Event struct { StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line - Waap AppsecEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"` + Appsec AppsecEvent `yaml:"Appsec,omitempty" json:"Appsec,omitempty"` /* Meta is the only part that will make it to the API - it should be normalized */ Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } From aa02a00fc268e8adaa22d4dba2a47493a5d1aedf Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 10:57:02 +0100 Subject: [PATCH 245/263] remove unused var --- pkg/acquisition/modules/appsec/appsec.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index d55c077fb78..db72f936785 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -50,7 +50,6 @@ type AppsecSource struct { logger *log.Entry mux *http.ServeMux server *http.Server - addr string outChan chan types.Event InChan chan appsec.ParsedRequest AppsecRuntime *appsec.AppsecRuntimeConfig From 91a6263b5b901da1c7a20c8c8139301b3cebe8de Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Tue, 5 Dec 2023 11:00:12 +0100 Subject: [PATCH 246/263] use official way of getting metrics for acquisition --- cmd/crowdsec/metrics.go | 4 ---- pkg/acquisition/modules/appsec/appsec.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index c9d88501a67..ca893872edb 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -12,7 +12,6 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/go-cs-lib/version" - appsec "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -163,8 +162,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, leaky.BucketsCurrentCount, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - appsec.AppsecGlobalParsingHistogram, appsec.AppsecReqCounter, appsec.AppsecRuleHits, - appsec.AppsecBlockCounter, ) } else { log.Infof("Loading prometheus collectors") @@ -175,7 +172,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - appsec.AppsecGlobalParsingHistogram, appsec.AppsecInbandParsingHistogram, appsec.AppsecOutbandParsingHistogram, appsec.AppsecReqCounter, appsec.AppsecRuleHits, appsec.AppsecBlockCounter, ) } diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index db72f936785..47ebf2e752d 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -139,11 +139,11 @@ func (wc *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { } func (w *AppsecSource) GetMetrics() []prometheus.Collector { - return nil + return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} } func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { - return nil + return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} } func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { From bd2c59b05420be040a297fb3b30cf11c18a6770a Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 13:55:49 +0100 Subject: [PATCH 247/263] fix some tests --- test/bats/20_hub.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 5189044b91b..200b3180a8f 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -34,9 +34,9 @@ teardown() { # no items rune -0 cscli hub list - assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" + assert_output --regexp "APPSEC-CONFIGS.*APPSEC-RULES.*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*" rune -0 cscli hub list -o json - assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' + assert_json '{appsec-configs:[],appsec-rules:[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw assert_output 'name,status,version,description,type' @@ -135,7 +135,7 @@ teardown() { assert_line "collections" rune -0 cscli hub types -o human rune -0 yq -o json <(output) - assert_json '["parsers","postoverflows","scenarios","collections"]' + assert_json '["appsec-configs","appsec-rules","parsers","postoverflows","scenarios","collections"]' rune -0 cscli hub types -o json - assert_json '["parsers","postoverflows","scenarios","collections"]' + assert_json '["appsec-configs","appsec-rules","parsers","postoverflows","scenarios","collections"]' } From 63f230b24b9f82e06785ccd1b219903165cff3fe Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Tue, 5 Dec 2023 14:49:25 +0100 Subject: [PATCH 248/263] remove hub-1.5.6 reference from github workflows --- .github/workflows/bats.yml | 2 -- .github/workflows/codeql-analysis.yml | 2 -- .github/workflows/docker-tests.yml | 2 -- .github/workflows/go-tests-windows.yml | 2 -- .github/workflows/go-tests.yml | 2 -- 5 files changed, 10 deletions(-) diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index fbe5094d184..0ce8cf041ed 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -15,14 +15,12 @@ on: push: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - "README.md" pull_request: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - "README.md" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d0f091e5cbd..ce94822747d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,13 +16,11 @@ on: push: branches: - master - - hub-1.5.6 - releases/** pull_request: # The branches below must be a subset of the branches above branches: - master - - hub-1.5.6 - releases/** schedule: - cron: '15 16 * * 2' diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 0589975c6d6..913c4766238 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -4,14 +4,12 @@ on: push: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 441f7b44b1c..868eae92afa 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -4,14 +4,12 @@ on: push: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index a2bb7a2a705..b55431e144a 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -9,14 +9,12 @@ on: push: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' pull_request: branches: - master - - hub-1.5.6 - releases/** paths-ignore: - 'README.md' From 9b79a37effd07dc746a13e094b314ba8672c3983 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 16:23:08 +0100 Subject: [PATCH 249/263] display crowdsec logs when nuclei tests fail --- pkg/hubtest/hubtest_item.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 14a022a6d55..b3df015ba36 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -605,13 +605,20 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { } err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget) - + crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir) if t.Config.ExpectedNucleiFailure { if err != nil && errors.Is(err, NucleiTemplateFail) { log.Infof("Appsec test %s failed as expected", t.Name) t.Success = true } else { log.Errorf("Appsec test %s failed: %s", t.Name, err) + crowdsecLog, err := os.ReadFile(crowdsecLogFile) + if err != nil { + log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) + } else { + log.Errorf("crowdsec log file '%s'", crowdsecLogFile) + log.Errorf("%s", string(crowdsecLog)) + } } } else { if err == nil { @@ -619,6 +626,13 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { t.Success = true } else { log.Errorf("Appsec test %s failed: %s", t.Name, err) + crowdsecLog, err := os.ReadFile(crowdsecLogFile) + if err != nil { + log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) + } else { + log.Errorf("crowdsec log file '%s'", crowdsecLogFile) + log.Errorf("%s", string(crowdsecLog)) + } } } crowdsecDaemon.Process.Kill() From 0c030a3bb559518be75e842447c4f7b962e01dc5 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 16:49:34 +0100 Subject: [PATCH 250/263] use fmt.Printf to make it more readable --- pkg/hubtest/hubtest_item.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index b3df015ba36..2ee9a7d7cfd 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -617,7 +617,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) } else { log.Errorf("crowdsec log file '%s'", crowdsecLogFile) - log.Errorf("%s", string(crowdsecLog)) + fmt.Printf("%s\n", string(crowdsecLog)) } } } else { @@ -631,7 +631,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) } else { log.Errorf("crowdsec log file '%s'", crowdsecLogFile) - log.Errorf("%s", string(crowdsecLog)) + fmt.Printf("%s\n", string(crowdsecLog)) } } } From f7c5726a0a0792b00fe97bca62d449ca546e4da3 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:06:25 +0100 Subject: [PATCH 251/263] minor reverts and tweaks (#2639) --- cmd/crowdsec-cli/main.go | 1 - pkg/csconfig/config.go | 3 --- pkg/csconfig/config_paths.go | 4 ---- pkg/cwhub/sync.go | 21 +++++++-------------- test/bats/20_hub.bats | 6 +++--- 5 files changed, 10 insertions(+), 25 deletions(-) diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index d5def40c7ed..b38d56ce95c 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -55,7 +55,6 @@ func initConfig() { log.Fatal(err) } } else { - // XXX: check all the defaults csConfig = csconfig.NewDefaultConfig() } diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 9e90c20cc88..09f46e25058 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -100,9 +100,6 @@ func GetConfig() Config { return globalConfig } -// XXX: We must not have a different behavior with an empty vs a missing configuration file. -// XXX: For this reason, all defaults have to come from NewConfig(). The following function should -// XXX: be replaced func NewDefaultConfig() *Config { logLevel := log.InfoLevel commonCfg := CommonCfg{ diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 07db4bd710c..71e3bacdaac 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -18,22 +18,18 @@ type ConfigurationPaths struct { func (c *Config) loadConfigurationPaths() error { var err error if c.ConfigPaths == nil { - // XXX: test me return fmt.Errorf("no configuration paths provided") } if c.ConfigPaths.DataDir == "" { - // XXX: test me return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section") } if c.ConfigPaths.HubDir == "" { - // XXX: test me c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub") } if c.ConfigPaths.HubIndexFile == "" { - // XXX: test me c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json") } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 72f97b114ef..ed3abc2c527 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "slices" "sort" "strings" @@ -112,21 +113,13 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) - if ret.stage == SCENARIOS { - ret.ftype = SCENARIOS - ret.stage = "" - } else if ret.stage == COLLECTIONS { - ret.ftype = COLLECTIONS - ret.stage = "" - } else if ret.stage == APPSEC_RULES { - ret.ftype = APPSEC_RULES - ret.stage = "" - } else if ret.stage == APPSEC_CONFIGS { - ret.ftype = APPSEC_CONFIGS + if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { + if !slices.Contains(ItemTypes, ret.stage) { + return nil, fmt.Errorf("unknown configuration type for file '%s'", path) + } + + ret.ftype = ret.stage ret.stage = "" - } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { - // it's a PARSER / POSTOVERFLOW with a stage - return nil, fmt.Errorf("unknown configuration type for file '%s'", path) } log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 3558e860485..0b222dde0a1 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -36,7 +36,7 @@ teardown() { rune -0 cscli hub list assert_output "No items to display" rune -0 cscli hub list -o json - assert_json '{appsec-configs:[],appsec-rules:[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' + assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw assert_output 'name,status,version,description,type' @@ -137,7 +137,7 @@ teardown() { assert_line "collections" rune -0 cscli hub types -o human rune -0 yq -o json <(output) - assert_json '["appsec-configs","appsec-rules","parsers","postoverflows","scenarios","collections"]' + assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' rune -0 cscli hub types -o json - assert_json '["appsec-configs","appsec-rules","parsers","postoverflows","scenarios","collections"]' + assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' } From 169e39a4a952eb0ccb9141ca8cfe7803e088c2a5 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 17:22:51 +0100 Subject: [PATCH 252/263] fix log level propagation + log requests to the appsec engine --- pkg/acquisition/modules/appsec/appsec.go | 10 +++------- pkg/appsec/appsec.go | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 47ebf2e752d..e30f0e35e96 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -97,11 +97,6 @@ func (wc *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { return errors.Wrap(err, "Cannot parse appsec configuration") } - if wc.config.LogLevel == nil { - level := new(log.Level) - *level = log.InfoLevel - wc.config.LogLevel = level - } if wc.config.ListenAddr == "" { wc.config.ListenAddr = "127.0.0.1:7422" } @@ -152,7 +147,6 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "unable to parse appsec configuration") } w.logger = logger - w.logger.Logger.SetLevel(*w.config.LogLevel) w.logger.Tracef("Appsec configuration: %+v", w.config) @@ -222,7 +216,6 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.appsecHandler) - return nil } @@ -316,6 +309,9 @@ func (w *AppsecSource) IsAuth(apiKey string) bool { // should this be in the runner ? func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { + + w.logger.Debugf("Received request from '%s' on %s", r.RemoteAddr, r.URL.Path) + apiKey := r.Header.Get(appsec.APIKeyHeaderName) clientIP := r.Header.Get(appsec.IPHeaderName) remoteIP := r.RemoteAddr diff --git a/pkg/appsec/appsec.go b/pkg/appsec/appsec.go index 7f8cba53ecc..8fe1412f743 100644 --- a/pkg/appsec/appsec.go +++ b/pkg/appsec/appsec.go @@ -157,10 +157,10 @@ func (wc *AppsecConfig) LoadByPath(file string) error { return fmt.Errorf("name cannot be empty") } if wc.LogLevel == nil { - lvl := log.InfoLevel + lvl := wc.Logger.Logger.GetLevel() wc.LogLevel = &lvl } - wc.Logger = wc.Logger.WithField("name", wc.Name) + wc.Logger = wc.Logger.Dup().WithField("name", wc.Name) wc.Logger.Logger.SetLevel(*wc.LogLevel) if wc.DefaultRemediation == "" { return fmt.Errorf("default_remediation cannot be empty") From 5503b2374afa4f14ed476f462cf04047bcd875f1 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 5 Dec 2023 17:32:03 +0100 Subject: [PATCH 253/263] up --- pkg/hubtest/hubtest_item.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 2ee9a7d7cfd..05be6803da7 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -617,7 +617,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) } else { log.Errorf("crowdsec log file '%s'", crowdsecLogFile) - fmt.Printf("%s\n", string(crowdsecLog)) + log.Errorf("%s\n", string(crowdsecLog)) } } } else { @@ -631,7 +631,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error { log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) } else { log.Errorf("crowdsec log file '%s'", crowdsecLogFile) - fmt.Printf("%s\n", string(crowdsecLog)) + log.Errorf("%s\n", string(crowdsecLog)) } } } From 25635a306f57cb99ec4ad8f7ec690fae2d186956 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 6 Dec 2023 10:27:29 +0100 Subject: [PATCH 254/263] propagate labels from acquis to appsec events --- pkg/acquisition/modules/appsec/appsec_runner.go | 5 +++-- pkg/acquisition/modules/appsec/utils.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 0887c725f2c..c778eb9b908 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -23,6 +23,7 @@ type AppsecRunner struct { AppsecRuntime *appsec.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc. AppsecInbandEngine coraza.WAF AppsecOutbandEngine coraza.WAF + Labels map[string]string logger *log.Entry } @@ -205,7 +206,7 @@ func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) erro func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { //create the associated event for crowdsec itself - evt, err := EventFromRequest(request) + evt, err := EventFromRequest(request, r.Labels) if err != nil { //let's not interrupt the pipeline for this r.logger.Errorf("unable to create event from request : %s", err) @@ -253,7 +254,7 @@ func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { } func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { - evt, err := EventFromRequest(request) + evt, err := EventFromRequest(request, r.Labels) if err != nil { //let's not interrupt the pipeline for this r.logger.Errorf("unable to create event from request : %s", err) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 3776265395b..2dcddc8b7c2 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -68,7 +68,7 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { return &evt, nil } -func EventFromRequest(r *appsec.ParsedRequest) (types.Event, error) { +func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -91,7 +91,7 @@ func EventFromRequest(r *appsec.ParsedRequest) (types.Event, error) { evt.Line = types.Line{ Time: time.Now(), //should we add some info like listen addr/port/path ? - Labels: map[string]string{"type": "crowdsec-appsec"}, //FIXME: use the labels from the acquis + Labels: labels, Process: true, Module: "appsec", Src: "appsec", From 00d899ee8ef0c8b7c335dd1ab9d0d129ed17cd0f Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 6 Dec 2023 10:35:04 +0100 Subject: [PATCH 255/263] rename struct in UnmarshalConfig --- pkg/acquisition/modules/appsec/appsec.go | 35 ++++++++++++------------ 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index e30f0e35e96..7f0a8155b4d 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -90,45 +90,45 @@ type BodyResponse struct { Action string `json:"action"` } -func (wc *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { +func (w *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { - err := yaml.UnmarshalStrict(yamlConfig, &wc.config) + err := yaml.UnmarshalStrict(yamlConfig, &w.config) if err != nil { return errors.Wrap(err, "Cannot parse appsec configuration") } - if wc.config.ListenAddr == "" { - wc.config.ListenAddr = "127.0.0.1:7422" + if w.config.ListenAddr == "" { + w.config.ListenAddr = "127.0.0.1:7422" } - if wc.config.Path == "" { - wc.config.Path = "/" + if w.config.Path == "" { + w.config.Path = "/" } - if wc.config.Path[0] != '/' { - wc.config.Path = "/" + wc.config.Path + if w.config.Path[0] != '/' { + w.config.Path = "/" + w.config.Path } - if wc.config.Mode == "" { - wc.config.Mode = configuration.TAIL_MODE + if w.config.Mode == "" { + w.config.Mode = configuration.TAIL_MODE } // always have at least one appsec routine - if wc.config.Routines == 0 { - wc.config.Routines = 1 + if w.config.Routines == 0 { + w.config.Routines = 1 } - if wc.config.AppsecConfig == "" && wc.config.AppsecConfigPath == "" { + if w.config.AppsecConfig == "" && w.config.AppsecConfigPath == "" { return fmt.Errorf("appsec_config or appsec_config_path must be set") } - if wc.config.Name == "" { - wc.config.Name = fmt.Sprintf("%s%s", wc.config.ListenAddr, wc.config.Path) + if w.config.Name == "" { + w.config.Name = fmt.Sprintf("%s%s", w.config.ListenAddr, w.config.Path) } csConfig := csconfig.GetConfig() - wc.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL) - wc.AuthCache = NewAuthCache() + w.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL) + w.AuthCache = NewAuthCache() return nil } @@ -204,6 +204,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { "uuid": appsecRunnerUUID, }), AppsecRuntime: &wrt, + Labels: w.config.Labels, } err := runner.Init(appsecCfg.GetDataDir()) if err != nil { From dce1f3cd8c8df3176b01dd4fb6042012974825c1 Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 6 Dec 2023 10:48:03 +0100 Subject: [PATCH 256/263] lower debug here, fix logging there --- .../modules/appsec/appsec_runner.go | 2 +- pkg/acquisition/modules/appsec/utils.go | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index c778eb9b908..bdc2c2080ff 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -197,7 +197,7 @@ func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { } func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { - r.logger.Infof("Processing out of band rules") + r.logger.Debugf("Processing out of band rules") tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) r.AppsecRuntime.OutOfBandTx = tx err := r.processRequest(tx, request) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 2dcddc8b7c2..7afa5a03bf4 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -53,16 +53,16 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.EventsCount = ptr.Of(int32(1)) alert.Labels = []string{"appsec"} //don't know what to do about this alert.Leakspeed = ptr.Of("") - msg := fmt.Sprintf("Application Security Engine alert: %s", inEvt.Appsec.MatchedRules.GetName()) - alert.Message = &msg alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName()) alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash()) alert.ScenarioVersion = ptr.Of(inEvt.Appsec.MatchedRules.GetVersion()) alert.Simulated = ptr.Of(false) alert.Source = &source + msg := fmt.Sprintf("AppSec block: %s from %s (%s)", inEvt.Appsec.MatchedRules.GetName(), + alert.Source.IP, inEvt.Parsed["remediation_cmpt_ip"]) + alert.Message = &msg alert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) alert.StopAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) - evt.Overflow.APIAlerts = []models.Alert{alert} evt.Overflow.Alert = &alert return &evt, nil @@ -76,13 +76,13 @@ func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types. //def needs fixing evt.Stage = "s00-raw" evt.Parsed = map[string]string{ - "source_ip": r.ClientIP, - "target_host": r.Host, - "target_uri": r.URI, - "method": r.Method, - "req_uuid": r.Tx.ID(), - "source": "crowdsec-appsec", - + "source_ip": r.ClientIP, + "target_host": r.Host, + "target_uri": r.URI, + "method": r.Method, + "req_uuid": r.Tx.ID(), + "source": "crowdsec-appsec", + "remediation_cmpt_ip": r.RemoteAddrNormalized, //TBD: //http_status //user_agent From c9e4aebd00e26d056b0f5f43c6e2df259476a2bf Mon Sep 17 00:00:00 2001 From: bui Date: Wed, 6 Dec 2023 10:54:28 +0100 Subject: [PATCH 257/263] up --- pkg/acquisition/modules/appsec/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 7afa5a03bf4..5d872f1df2a 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -51,7 +51,6 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert.Meta = append(alert.Meta, &meta) } alert.EventsCount = ptr.Of(int32(1)) - alert.Labels = []string{"appsec"} //don't know what to do about this alert.Leakspeed = ptr.Of("") alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName()) alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash()) From 0c617269716f066cb31d84dafc554c093cb9aa28 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 6 Dec 2023 11:21:54 +0100 Subject: [PATCH 258/263] propagate request_id/runner_id in more places for logging --- pkg/acquisition/modules/appsec/appsec.go | 13 +++++++++---- pkg/acquisition/modules/appsec/appsec_runner.go | 8 +++++--- pkg/appsec/appsec.go | 6 +++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 7f0a8155b4d..2ae0ad939c8 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -197,11 +197,12 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { appsecRunnerUUID := uuid.New().String() //we copy AppsecRutime for each runner wrt := *w.AppsecRuntime + wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID) runner := AppsecRunner{ inChan: w.InChan, UUID: appsecRunnerUUID, logger: w.logger.WithFields(log.Fields{ - "uuid": appsecRunnerUUID, + "runner_uuid": appsecRunnerUUID, }), AppsecRuntime: &wrt, Labels: w.config.Labels, @@ -310,7 +311,6 @@ func (w *AppsecSource) IsAuth(apiKey string) bool { // should this be in the runner ? func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { - w.logger.Debugf("Received request from '%s' on %s", r.RemoteAddr, r.URL.Path) apiKey := r.Header.Get(appsec.APIKeyHeaderName) @@ -343,6 +343,11 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { } parsedRequest.AppsecEngine = w.config.Name + logger := w.logger.WithFields(log.Fields{ + "request_uuid": parsedRequest.UUID, + "client_ip": parsedRequest.ClientIP, + }) + AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() w.InChan <- parsedRequest @@ -352,12 +357,12 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() } - appsecResponse := w.AppsecRuntime.GenerateResponse(response) + appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger) rw.WriteHeader(appsecResponse.HTTPStatus) body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action}) if err != nil { - log.Errorf("unable to marshal response: %s", err) + logger.Errorf("unable to marshal response: %s", err) rw.WriteHeader(http.StatusInternalServerError) } else { rw.Write(body) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index bdc2c2080ff..086060b4657 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -290,7 +290,9 @@ func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { } func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { - r.logger.Debugf("Requests handled by runner %s", request.UUID) + r.AppsecRuntime.Logger = r.AppsecRuntime.Logger.WithField("request_uuid", request.UUID) + logger := r.logger.WithField("request_uuid", request.UUID) + logger.Debug("Request received in runner") r.AppsecRuntime.ClearResponse() request.IsInBand = true @@ -302,7 +304,7 @@ func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { //inband appsec rules err := r.ProcessInBandRules(request) if err != nil { - r.logger.Errorf("unable to process InBand rules: %s", err) + logger.Errorf("unable to process InBand rules: %s", err) return } @@ -325,7 +327,7 @@ func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { err = r.ProcessOutOfBandRules(request) if err != nil { - r.logger.Errorf("unable to process OutOfBand rules: %s", err) + logger.Errorf("unable to process OutOfBand rules: %s", err) return } diff --git a/pkg/appsec/appsec.go b/pkg/appsec/appsec.go index 8fe1412f743..4550990b200 100644 --- a/pkg/appsec/appsec.go +++ b/pkg/appsec/appsec.go @@ -556,7 +556,7 @@ type BodyResponse struct { HTTPStatus int `json:"http_status"` } -func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse) BodyResponse { +func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse, logger *log.Entry) BodyResponse { resp := BodyResponse{} //if there is no interrupt, we should allow with default code if !response.InBandInterrupt { @@ -568,12 +568,12 @@ func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse) Body if resp.Action == "" { resp.Action = w.Config.DefaultRemediation } - w.Logger.Debugf("action is %s", resp.Action) + logger.Debugf("action is %s", resp.Action) resp.HTTPStatus = response.HTTPResponseCode if resp.HTTPStatus == 0 { resp.HTTPStatus = w.Config.BlockedHTTPCode } - w.Logger.Debugf("http status is %d", resp.HTTPStatus) + logger.Debugf("http status is %d", resp.HTTPStatus) return resp } From fe78511b48e01257fa7e35b9bb09f91cddd483d9 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:09:27 +0100 Subject: [PATCH 259/263] cscli: simplify generic item commands (#2641) --- cmd/crowdsec-cli/appsec_rules.go | 10 +- cmd/crowdsec-cli/itemcommands.go | 439 +++++++++++++++---------------- 2 files changed, 210 insertions(+), 239 deletions(-) diff --git a/cmd/crowdsec-cli/appsec_rules.go b/cmd/crowdsec-cli/appsec_rules.go index 9fa2f85d6e9..7980368f2b0 100644 --- a/cmd/crowdsec-cli/appsec_rules.go +++ b/cmd/crowdsec-cli/appsec_rules.go @@ -64,7 +64,7 @@ func NewCmdAppsecRulesInstall() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsInstallRunner(hubItemTypes[cwhub.APPSEC_RULES]), + RunE: hubItemTypes[cwhub.APPSEC_RULES].Install, } flags := cmdAppsecRulesInstall.Flags() @@ -86,7 +86,7 @@ func NewCmdAppsecRulesRemove() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsRemoveRunner(hubItemTypes[cwhub.APPSEC_RULES]), + RunE: hubItemTypes[cwhub.APPSEC_RULES].Remove, } flags := cmdAppsecRulesRemove.Flags() @@ -107,7 +107,7 @@ func NewCmdAppsecRulesUpgrade() *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) }, - RunE: itemsUpgradeRunner(hubItemTypes[cwhub.APPSEC_RULES]), + RunE: hubItemTypes[cwhub.APPSEC_RULES].Upgrade, } flags := cmdAppsecRulesUpgrade.Flags() @@ -119,7 +119,7 @@ func NewCmdAppsecRulesUpgrade() *cobra.Command { func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - f := itemsInspectRunner(hubItemTypes[cwhub.APPSEC_RULES]) + f := hubItemTypes[cwhub.APPSEC_RULES].Inspect if err := f(cmd, args); err != nil { return err } @@ -182,7 +182,7 @@ func NewCmdAppsecRulesList() *cobra.Command { cscli appsec-rules list -a cscli appsec-rules list crowdsecurity/crs`, DisableAutoGenTag: true, - RunE: itemsListRunner(hubItemTypes[cwhub.APPSEC_RULES]), + RunE: hubItemTypes[cwhub.APPSEC_RULES].List, } flags := cmdAppsecRulesList.Flags() diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 8003486fdf0..a58042c089e 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -238,70 +238,64 @@ func NewItemsCmd(typeName string) *cobra.Command { DisableAutoGenTag: true, } - cmd.AddCommand(NewItemsInstallCmd(typeName)) - cmd.AddCommand(NewItemsRemoveCmd(typeName)) - cmd.AddCommand(NewItemsUpgradeCmd(typeName)) - cmd.AddCommand(NewItemsInspectCmd(typeName)) - cmd.AddCommand(NewItemsListCmd(typeName)) + cmd.AddCommand(it.NewInstallCmd()) + cmd.AddCommand(it.NewRemoveCmd()) + cmd.AddCommand(it.NewUpgradeCmd()) + cmd.AddCommand(it.NewInspectCmd()) + cmd.AddCommand(it.NewListCmd()) return cmd } -func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() +func (it hubItemType) Install(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } - force, err := flags.GetBool("force") - if err != nil { - return err - } + force, err := flags.GetBool("force") + if err != nil { + return err + } - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) - if err != nil { - return err - } + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } - for _, name := range args { - item := hub.GetItem(it.name, name) - if item == nil { - msg := suggestNearestMessage(hub, it.name, name) - if !ignoreError { - return fmt.Errorf(msg) - } + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + msg := suggestNearestMessage(hub, it.name, name) + if !ignoreError { + return fmt.Errorf(msg) + } - log.Errorf(msg) + log.Errorf(msg) - continue - } + continue + } - if err := item.Install(force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", item.Name, err) - } - log.Errorf("Error while installing '%s': %s", item.Name, err) + if err := item.Install(force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", item.Name, err) } + log.Errorf("Error while installing '%s': %s", item.Name, err) } - - log.Infof(ReloadMessage()) - return nil } - return run + log.Infof(ReloadMessage()) + return nil } -func NewItemsInstallCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - +func (it hubItemType) NewInstallCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.installHelp.use, "install [item]..."), Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), @@ -310,9 +304,9 @@ func NewItemsInstallCmd(typeName string) *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(typeName, args, toComplete) + return compAllItems(it.name, args, toComplete) }, - RunE: itemsInstallRunner(it), + RunE: it.Install, } flags := cmd.Flags() @@ -336,87 +330,47 @@ func istalledParentNames(item *cwhub.Item) []string { return ret } -func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } +func (it hubItemType) Remove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - if all { - getter := hub.GetInstalledItems - if purge { - getter = hub.GetAllItems - } + purge, err := flags.GetBool("purge") + if err != nil { + return err + } - items, err := getter(it.name) - if err != nil { - return err - } + force, err := flags.GetBool("force") + if err != nil { + return err + } - removed := 0 - - for _, item := range items { - didRemove, err := item.Remove(purge, force) - if err != nil { - return err - } - if didRemove { - log.Infof("Removed %s", item.Name) - removed++ - } - } + all, err := flags.GetBool("all") + if err != nil { + return err + } - log.Infof("Removed %d %s", removed, it.name) - if removed > 0 { - log.Infof(ReloadMessage()) - } + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } - return nil + if all { + getter := hub.GetInstalledItems + if purge { + getter = hub.GetAllItems } - if len(args) == 0 { - return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + items, err := getter(it.name) + if err != nil { + return err } removed := 0 - for _, itemName := range args { - item := hub.GetItem(it.name, itemName) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, it.name) - } - - parents := istalledParentNames(item) - - if !force && len(parents) > 0 { - log.Warningf("%s belongs to collections: %s", item.Name, parents) - log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) - continue - } - + for _, item := range items { didRemove, err := item.Remove(purge, force) if err != nil { return err } - if didRemove { log.Infof("Removed %s", item.Name) removed++ @@ -430,12 +384,47 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e return nil } - return run -} -func NewItemsRemoveCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + } + + removed := 0 + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + parents := istalledParentNames(item) + + if !force && len(parents) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, parents) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + continue + } + + didRemove, err := item.Remove(purge, force) + if err != nil { + return err + } + + if didRemove { + log.Infof("Removed %s", item.Name) + removed++ + } + } + + log.Infof("Removed %d %s", removed, it.name) + if removed > 0 { + log.Infof(ReloadMessage()) + } + + return nil +} + +func (it hubItemType) NewRemoveCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.removeHelp.use, "remove [item]..."), Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), @@ -446,7 +435,7 @@ func NewItemsRemoveCmd(typeName string) *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(it.name, args, toComplete) }, - RunE: itemsRemoveRunner(it), + RunE: it.Remove, } flags := cmd.Flags() @@ -457,74 +446,44 @@ func NewItemsRemoveCmd(typeName string) *cobra.Command { return cmd } -func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() +func (it hubItemType) Upgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - force, err := flags.GetBool("force") - if err != nil { - return err - } + force, err := flags.GetBool("force") + if err != nil { + return err + } - all, err := flags.GetBool("all") - if err != nil { - return err - } + all, err := flags.GetBool("all") + if err != nil { + return err + } - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + if all { + items, err := hub.GetInstalledItems(it.name) if err != nil { return err } - if all { - items, err := hub.GetInstalledItems(it.name) - if err != nil { - return err - } - - updated := 0 - - for _, item := range items { - didUpdate, err := item.Upgrade(force) - if err != nil { - return err - } - if didUpdate { - updated++ - } - } - - log.Infof("Updated %d %s", updated, it.name) - - if updated > 0 { - log.Infof(ReloadMessage()) - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) - } - updated := 0 - for _, itemName := range args { - item := hub.GetItem(it.name, itemName) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, it.name) - } - + for _, item := range items { didUpdate, err := item.Upgrade(force) if err != nil { return err } - if didUpdate { - log.Infof("Updated %s", item.Name) updated++ } } + + log.Infof("Updated %d %s", updated, it.name) + if updated > 0 { log.Infof(ReloadMessage()) } @@ -532,12 +491,36 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) return nil } - return run -} + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) + } -func NewItemsUpgradeCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] + updated := 0 + + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + + if didUpdate { + log.Infof("Updated %s", item.Name) + updated++ + } + } + if updated > 0 { + log.Infof(ReloadMessage()) + } + + return nil +} + +func (it hubItemType) NewUpgradeCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), @@ -547,7 +530,7 @@ func NewItemsUpgradeCmd(typeName string) *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(it.name, args, toComplete) }, - RunE: itemsUpgradeRunner(it), + RunE: it.Upgrade, } flags := cmd.Flags() @@ -557,48 +540,42 @@ func NewItemsUpgradeCmd(typeName string) *cobra.Command { return cmd } -func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() +func (it hubItemType) Inspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - url, err := flags.GetString("url") - if err != nil { - return err - } + url, err := flags.GetString("url") + if err != nil { + return err + } - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } - for _, name := range args { - item := hub.GetItem(it.name, name) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", name, it.name) - } - if err = InspectItem(item, !noMetrics); err != nil { - return err - } + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if err = InspectItem(item, !noMetrics); err != nil { + return err } - - return nil } - return run + return nil } -func NewItemsInspectCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - +func (it hubItemType) NewInspectCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), @@ -609,7 +586,7 @@ func NewItemsInspectCmd(typeName string) *cobra.Command { ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(it.name, args, toComplete) }, - RunE: itemsInspectRunner(it), + RunE: it.Inspect, } flags := cmd.Flags() @@ -619,47 +596,41 @@ func NewItemsInspectCmd(typeName string) *cobra.Command { return cmd } -func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } +func (it hubItemType) List(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } + all, err := flags.GetBool("all") + if err != nil { + return err + } - items := make(map[string][]*cwhub.Item) + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } - items[it.name], err = selectItems(hub, it.name, args, !all) - if err != nil { - return err - } + items := make(map[string][]*cwhub.Item) - if err = listItems(color.Output, []string{it.name}, items, false); err != nil { - return err - } + items[it.name], err = selectItems(hub, it.name, args, !all) + if err != nil { + return err + } - return nil + if err = listItems(color.Output, []string{it.name}, items, false); err != nil { + return err } - return run + return nil } -func NewItemsListCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - +func (it hubItemType) NewListCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), Example: it.listHelp.example, DisableAutoGenTag: true, - RunE: itemsListRunner(it), + RunE: it.List, } flags := cmd.Flags() From 493880824b7013772ec7934330456902e8886a3d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 6 Dec 2023 13:23:56 +0100 Subject: [PATCH 260/263] add matched zones in context for appsec alerts --- pkg/acquisition/modules/appsec/utils.go | 72 ++++++++++++++++++------- pkg/types/appsec_event.go | 9 ++++ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 5d872f1df2a..8d879912c1a 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -34,8 +34,8 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { alert := models.Alert{} alert.Capacity = ptr.Of(int32(1)) - alert.Events = make([]*models.Event, 0) //@tko -> URI, method, UA, param name - alert.Meta = make(models.Meta, 0) //@tko -> URI, method, UA, param name + alert.Events = make([]*models.Event, 0) + alert.Meta = make(models.Meta, 0) for _, key := range []string{"target_uri", "method"} { valueByte, err := json.Marshal([]string{inEvt.Parsed[key]}) @@ -50,6 +50,31 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { } alert.Meta = append(alert.Meta, &meta) } + matchedZones := inEvt.Appsec.GetMatchedZones() + if matchedZones != nil { + valueByte, err := json.Marshal(matchedZones) + if err != nil { + log.Debugf("unable to serialize key matched_zones") + } else { + meta := models.MetaItems0{ + Key: "matched_zones", + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + } + } + for _, key := range evt.Appsec.MatchedRules.GetMatchedZones() { + valueByte, err := json.Marshal([]string{key}) + if err != nil { + log.Debugf("unable to serialize key %s", key) + continue + } + meta := models.MetaItems0{ + Key: "matched_zones", + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + } alert.EventsCount = ptr.Of(int32(1)) alert.Leakspeed = ptr.Of("") alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName()) @@ -216,23 +241,34 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc() + matchedZones := make([]string, 0) + for _, matchData := range rule.MatchedDatas() { + zone := matchData.Variable().Name() + varName := matchData.Key() + if varName != "" { + zone += "." + varName + } + matchedZones = append(matchedZones, zone) + } + corazaRule := map[string]interface{}{ - "id": rule.Rule().ID(), - "uri": evt.Parsed["uri"], - "rule_type": kind, - "method": evt.Parsed["method"], - "disruptive": rule.Disruptive(), - "tags": rule.Rule().Tags(), - "file": rule.Rule().File(), - "file_line": rule.Rule().Line(), - "revision": rule.Rule().Revision(), - "secmark": rule.Rule().SecMark(), - "accuracy": rule.Rule().Accuracy(), - "msg": rule.Message(), - "severity": rule.Rule().Severity().String(), - "name": name, - "hash": hash, - "version": version, + "id": rule.Rule().ID(), + "uri": evt.Parsed["uri"], + "rule_type": kind, + "method": evt.Parsed["method"], + "disruptive": rule.Disruptive(), + "tags": rule.Rule().Tags(), + "file": rule.Rule().File(), + "file_line": rule.Rule().Line(), + "revision": rule.Rule().Revision(), + "secmark": rule.Rule().SecMark(), + "accuracy": rule.Rule().Accuracy(), + "msg": rule.Message(), + "severity": rule.Rule().Severity().String(), + "name": name, + "hash": hash, + "version": version, + "matched_zones": matchedZones, } evt.Appsec.MatchedRules = append(evt.Appsec.MatchedRules, corazaRule) } diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go index b31dfefefea..4cd5d8f585f 100644 --- a/pkg/types/appsec_event.go +++ b/pkg/types/appsec_event.go @@ -128,6 +128,15 @@ func (w MatchedRules) Kinds() []string { return ret } +func (w MatchedRules) GetMatchedZones() []string { + ret := make([]string, 0) + + for _, rule := range w { + ret = append(ret, rule["matched_zones"].([]string)...) + } + return ret +} + // filters func (w MatchedRules) ByID(id int) MatchedRules { ret := MatchedRules{} From 8fa84e5cd97f8b3a1457af0f4eed9dd8574b2f58 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:42:14 +0100 Subject: [PATCH 261/263] cscli: generic hubappsec (#2642) --- cmd/crowdsec-cli/appsec_rules.go | 192 -------------- cmd/crowdsec-cli/hubappsec.go | 105 ++++++++ cmd/crowdsec-cli/hubcollection.go | 40 +++ cmd/crowdsec-cli/hubparser.go | 40 +++ cmd/crowdsec-cli/hubpostoverflow.go | 40 +++ cmd/crowdsec-cli/hubscenario.go | 40 +++ .../{itemcommands.go => itemcli.go} | 244 +++--------------- cmd/crowdsec-cli/main.go | 13 +- 8 files changed, 301 insertions(+), 413 deletions(-) delete mode 100644 cmd/crowdsec-cli/appsec_rules.go create mode 100644 cmd/crowdsec-cli/hubappsec.go create mode 100644 cmd/crowdsec-cli/hubcollection.go create mode 100644 cmd/crowdsec-cli/hubparser.go create mode 100644 cmd/crowdsec-cli/hubpostoverflow.go create mode 100644 cmd/crowdsec-cli/hubscenario.go rename cmd/crowdsec-cli/{itemcommands.go => itemcli.go} (57%) diff --git a/cmd/crowdsec-cli/appsec_rules.go b/cmd/crowdsec-cli/appsec_rules.go deleted file mode 100644 index 7980368f2b0..00000000000 --- a/cmd/crowdsec-cli/appsec_rules.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "fmt" - "os" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "golang.org/x/text/cases" - "golang.org/x/text/language" - "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/appsec" - "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -func NewAppsecRulesCmd() *cobra.Command { - cmdAppsecRules := &cobra.Command{ - Use: "appsec-rules [appsec-rule]...", - Short: "Manage hub appsec rules", - Example: `cscli appsec-rules list -a -cscli appsec-rules install crowdsecurity/crs -cscli appsec-rules inspect crowdsecurity/crs -cscli appsec-rules upgrade crowdsecurity/crs -cscli appsec-rules remove crowdsecurity/crs -`, - Args: cobra.MinimumNArgs(1), - Aliases: []string{"appsec-rule"}, - DisableAutoGenTag: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if _, err := require.Hub(csConfig, require.RemoteHub(csConfig)); err != nil { - return err - } - - return nil - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - if cmd.Name() == "inspect" || cmd.Name() == "list" { - return - } - log.Infof(ReloadMessage()) - }, - } - - cmdAppsecRules.AddCommand(NewCmdAppsecRulesInstall()) - cmdAppsecRules.AddCommand(NewCmdAppsecRulesRemove()) - cmdAppsecRules.AddCommand(NewCmdAppsecRulesUpgrade()) - cmdAppsecRules.AddCommand(NewCmdAppsecRulesInspect()) - cmdAppsecRules.AddCommand(NewCmdAppsecRulesList()) - - return cmdAppsecRules -} - -func NewCmdAppsecRulesInstall() *cobra.Command { - cmdAppsecRulesInstall := &cobra.Command{ - Use: "install ...", - Short: "Install given appsec rule(s)", - Long: `Fetch and install one or more appsec rules from the hub`, - Example: `cscli appsec-rules install crowdsecurity/crs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.APPSEC_RULES, args, toComplete) - }, - RunE: hubItemTypes[cwhub.APPSEC_RULES].Install, - } - - flags := cmdAppsecRulesInstall.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, "Ignore errors when installing multiple appsec rules") - - return cmdAppsecRulesInstall -} - -func NewCmdAppsecRulesRemove() *cobra.Command { - cmdAppsecRulesRemove := &cobra.Command{ - Use: "remove ...", - Short: "Remove given appsec rule(s)", - Long: `remove one or more appsec rules`, - Example: `cscli appsec-rules remove crowdsecurity/crs`, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) - }, - RunE: hubItemTypes[cwhub.APPSEC_RULES].Remove, - } - - flags := cmdAppsecRulesRemove.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, "Remove all the appsec rules") - - return cmdAppsecRulesRemove -} - -func NewCmdAppsecRulesUpgrade() *cobra.Command { - cmdAppsecRulesUpgrade := &cobra.Command{ - Use: "upgrade ...", - Short: "Upgrade given appsec rule(s)", - Long: `Fetch and upgrade one or more appsec rules from the hub`, - Example: `cscli appsec-rules upgrade crowdsecurity/crs`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) - }, - RunE: hubItemTypes[cwhub.APPSEC_RULES].Upgrade, - } - - flags := cmdAppsecRulesUpgrade.Flags() - flags.BoolP("all", "a", false, "Upgrade all the appsec rules") - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmdAppsecRulesUpgrade -} - -func AppsecRulesInspectRunner(itemType hubItemType) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - f := hubItemTypes[cwhub.APPSEC_RULES].Inspect - if err := f(cmd, args); err != nil { - return err - } - if csConfig.Cscli.Output == "human" { - hub, _ := require.Hub(csConfig, nil) - for _, name := range args { - hubItem := hub.GetItem(itemType.name, name) - appsecRule := appsec.AppsecCollectionConfig{} - yamlContent, err := os.ReadFile(hubItem.State.LocalPath) - if err != nil { - return fmt.Errorf("unable to read file %s : %s", hubItem.State.LocalPath, err) - } - if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { - return fmt.Errorf("unable to unmarshal yaml file %s : %s", hubItem.State.LocalPath, err) - } - - for _, ruleType := range appsec_rule.SupportedTypes() { - fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) - for _, rule := range appsecRule.Rules { - convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) - if err != nil { - return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) - } - fmt.Println(convertedRule) - } - } - } - } - return nil - } -} - -func NewCmdAppsecRulesInspect() *cobra.Command { - cmdAppsecRulesInspect := &cobra.Command{ - Use: "inspect ", - Short: "Inspect a appsec rule", - Long: `Inspect a appsec rule`, - Example: `cscli appsec-rules inspect crowdsecurity/crs`, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.APPSEC_RULES, args, toComplete) - }, - RunE: AppsecRulesInspectRunner(hubItemTypes[cwhub.APPSEC_RULES]), - } - - flags := cmdAppsecRulesInspect.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmdAppsecRulesInspect -} - -func NewCmdAppsecRulesList() *cobra.Command { - cmdAppsecRulesList := &cobra.Command{ - Use: "list [appsec-rule]...", - Short: "List appsec rules", - Long: `List of installed/available/specified appsec rules`, - Example: `cscli appsec-rules list -cscli appsec-rules list -a -cscli appsec-rules list crowdsecurity/crs`, - DisableAutoGenTag: true, - RunE: hubItemTypes[cwhub.APPSEC_RULES].List, - } - - flags := cmdAppsecRulesList.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmdAppsecRulesList -} diff --git a/cmd/crowdsec-cli/hubappsec.go b/cmd/crowdsec-cli/hubappsec.go new file mode 100644 index 00000000000..1fc8b6c0048 --- /dev/null +++ b/cmd/crowdsec-cli/hubappsec.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewAppsecConfigCLI() *itemCLI { + return &itemCLI{ + name: cwhub.APPSEC_CONFIGS, + singular: "appsec-config", + oneOrMore: "appsec-config(s)", + help: cliHelp{ + example: `cscli appsec-configs list -a +cscli appsec-configs install crowdsecurity/vpatch +cscli appsec-configs inspect crowdsecurity/vpatch +cscli appsec-configs upgrade crowdsecurity/vpatch +cscli appsec-configs remove crowdsecurity/vpatch +`, + }, + installHelp: cliHelp{ + example: `cscli appsec-configs install crowdsecurity/vpatch`, + }, + removeHelp: cliHelp{ + example: `cscli appsec-configs remove crowdsecurity/vpatch`, + }, + upgradeHelp: cliHelp{ + example: `cscli appsec-configs upgrade crowdsecurity/vpatch`, + }, + inspectHelp: cliHelp{ + example: `cscli appsec-configs inspect crowdsecurity/vpatch`, + }, + listHelp: cliHelp{ + example: `cscli appsec-configs list +cscli appsec-configs list -a +cscli appsec-configs list crowdsecurity/vpatch`, + }, + } +} + +func NewAppsecRuleCLI() *itemCLI { + inspectDetail := func(item *cwhub.Item) error { + appsecRule := appsec.AppsecCollectionConfig{} + yamlContent, err := os.ReadFile(item.State.LocalPath) + if err != nil { + return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err) + } + if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { + return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err) + } + + for _, ruleType := range appsec_rule.SupportedTypes() { + fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) + for _, rule := range appsecRule.Rules { + convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) + if err != nil { + return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) + } + fmt.Println(convertedRule) + } + } + + return nil + } + + return &itemCLI{ + name: "appsec-rules", + singular: "appsec-rule", + oneOrMore: "appsec-rule(s)", + help: cliHelp{ + example: `cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs +`, + }, + installHelp: cliHelp{ + example: `cscli appsec-rules install crowdsecurity/crs`, + }, + removeHelp: cliHelp{ + example: `cscli appsec-rules remove crowdsecurity/crs`, + }, + upgradeHelp: cliHelp{ + example: `cscli appsec-rules upgrade crowdsecurity/crs`, + }, + inspectHelp: cliHelp{ + example: `cscli appsec-rules inspect crowdsecurity/crs`, + }, + inspectDetail: inspectDetail, + listHelp: cliHelp{ + example: `cscli appsec-rules list +cscli appsec-rules list -a +cscli appsec-rules list crowdsecurity/crs`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubcollection.go b/cmd/crowdsec-cli/hubcollection.go new file mode 100644 index 00000000000..a869479750f --- /dev/null +++ b/cmd/crowdsec-cli/hubcollection.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewCollectionCLI() *itemCLI { + return &itemCLI{ + name: cwhub.COLLECTIONS, + singular: "collection", + oneOrMore: "collection(s)", + help: cliHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cliHelp{ + example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + }, + removeHelp: cliHelp{ + example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, + }, + upgradeHelp: cliHelp{ + example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, + }, + inspectHelp: cliHelp{ + example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, + }, + listHelp: cliHelp{ + example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables + +List only enabled collections unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubparser.go b/cmd/crowdsec-cli/hubparser.go new file mode 100644 index 00000000000..d2af0b6fb53 --- /dev/null +++ b/cmd/crowdsec-cli/hubparser.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewParserCLI() *itemCLI { + return &itemCLI{ + name: cwhub.PARSERS, + singular: "parser", + oneOrMore: "parser(s)", + help: cliHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cliHelp{ + example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + removeHelp: cliHelp{ + example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + upgradeHelp: cliHelp{ + example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + inspectHelp: cliHelp{ + example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, + }, + listHelp: cliHelp{ + example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +List only enabled parsers unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubpostoverflow.go b/cmd/crowdsec-cli/hubpostoverflow.go new file mode 100644 index 00000000000..326c3fec524 --- /dev/null +++ b/cmd/crowdsec-cli/hubpostoverflow.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewPostOverflowCLI() *itemCLI { + return &itemCLI{ + name: cwhub.POSTOVERFLOWS, + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cliHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cliHelp{ + example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + removeHelp: cliHelp{ + example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + upgradeHelp: cliHelp{ + example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + inspectHelp: cliHelp{ + example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + listHelp: cliHelp{ + example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns + +List only enabled postoverflows unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubscenario.go b/cmd/crowdsec-cli/hubscenario.go new file mode 100644 index 00000000000..14b246f0f30 --- /dev/null +++ b/cmd/crowdsec-cli/hubscenario.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewScenarioCLI() *itemCLI { + return &itemCLI{ + name: cwhub.SCENARIOS, + singular: "scenario", + oneOrMore: "scenario(s)", + help: cliHelp{ + example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing +`, + }, + installHelp: cliHelp{ + example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + removeHelp: cliHelp{ + example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + upgradeHelp: cliHelp{ + example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + inspectHelp: cliHelp{ + example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + listHelp: cliHelp{ + example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing + +List only enabled scenarios unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcli.go similarity index 57% rename from cmd/crowdsec-cli/itemcommands.go rename to cmd/crowdsec-cli/itemcli.go index a58042c089e..0870fdeb4ac 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcli.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -type cmdHelp struct { +type cliHelp struct { // Example is required, the others have a default value // generated from the item type use string @@ -22,212 +22,20 @@ type cmdHelp struct { example string } -type hubItemType struct { - name string // plural, as used in the hub index - singular string - oneOrMore string // parenthetical pluralizaion: "parser(s)" - help cmdHelp - installHelp cmdHelp - removeHelp cmdHelp - upgradeHelp cmdHelp - inspectHelp cmdHelp - listHelp cmdHelp +type itemCLI struct { + name string // plural, as used in the hub index + singular string + oneOrMore string // parenthetical pluralizaion: "parser(s)" + help cliHelp + installHelp cliHelp + removeHelp cliHelp + upgradeHelp cliHelp + inspectHelp cliHelp + inspectDetail func(item *cwhub.Item) error + listHelp cliHelp } -var hubItemTypes = map[string]hubItemType{ - "parsers": { - name: cwhub.PARSERS, - singular: "parser", - oneOrMore: "parser(s)", - help: cmdHelp{ - example: `cscli parsers list -a -cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -`, - }, - installHelp: cmdHelp{ - example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - removeHelp: cmdHelp{ - example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - upgradeHelp: cmdHelp{ - example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - inspectHelp: cmdHelp{ - example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, - }, - listHelp: cmdHelp{ - example: `cscli parsers list -cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs - -List only enabled parsers unless "-a" or names are specified.`, - }, - }, - "postoverflows": { - name: cwhub.POSTOVERFLOWS, - singular: "postoverflow", - oneOrMore: "postoverflow(s)", - help: cmdHelp{ - example: `cscli postoverflows list -a -cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -`, - }, - installHelp: cmdHelp{ - example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - removeHelp: cmdHelp{ - example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - upgradeHelp: cmdHelp{ - example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - inspectHelp: cmdHelp{ - example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - listHelp: cmdHelp{ - example: `cscli postoverflows list -cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns - -List only enabled postoverflows unless "-a" or names are specified.`, - }, - }, - "scenarios": { - name: cwhub.SCENARIOS, - singular: "scenario", - oneOrMore: "scenario(s)", - help: cmdHelp{ - example: `cscli scenarios list -a -cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing -`, - }, - installHelp: cmdHelp{ - example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - removeHelp: cmdHelp{ - example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - upgradeHelp: cmdHelp{ - example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - inspectHelp: cmdHelp{ - example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - listHelp: cmdHelp{ - example: `cscli scenarios list -cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing - -List only enabled scenarios unless "-a" or names are specified.`, - }, - }, - "appsec-rules": { - name: "appsec-rules", - singular: "appsec-rule", - oneOrMore: "appsec-rule(s)", - help: cmdHelp{ - example: `cscli appsec-rules list -a -cscli appsec-rules install crowdsecurity/crs -cscli appsec-rules inspect crowdsecurity/crs -cscli appsec-rules upgrade crowdsecurity/crs -cscli appsec-rules remove crowdsecurity/crs -`, - }, - installHelp: cmdHelp{ - example: `cscli appsec-rules install crowdsecurity/crs`, - }, - removeHelp: cmdHelp{ - example: `cscli appsec-rules remove crowdsecurity/crs`, - }, - upgradeHelp: cmdHelp{ - example: `cscli appsec-rules upgrade crowdsecurity/crs`, - }, - inspectHelp: cmdHelp{ - example: `cscli appsec-rules inspect crowdsecurity/crs`, - }, - listHelp: cmdHelp{ - example: `cscli appsec-rules list -cscli appsec-rules list -a -cscli appsec-rules list crowdsecurity/crs`, - }, - }, - "appsec-configs": { - name: "appsec-configs", - singular: "appsec-config", - oneOrMore: "appsec-config(s)", - help: cmdHelp{ - example: `cscli appsec-configs list -a -cscli appsec-configs install crowdsecurity/vpatch -cscli appsec-configs inspect crowdsecurity/vpatch -cscli appsec-configs upgrade crowdsecurity/vpatch -cscli appsec-configs remove crowdsecurity/vpatch -`, - }, - installHelp: cmdHelp{ - example: `cscli appsec-configs install crowdsecurity/vpatch`, - }, - removeHelp: cmdHelp{ - example: `cscli appsec-configs remove crowdsecurity/vpatch`, - }, - upgradeHelp: cmdHelp{ - example: `cscli appsec-configs upgrade crowdsecurity/vpatch`, - }, - inspectHelp: cmdHelp{ - example: `cscli appsec-configs inspect crowdsecurity/vpatch`, - }, - listHelp: cmdHelp{ - example: `cscli appsec-configs list -cscli appsec-configs list -a -cscli appsec-configs list crowdsecurity/vpatch`, - }, - }, - "collections": { - name: cwhub.COLLECTIONS, - singular: "collection", - oneOrMore: "collection(s)", - help: cmdHelp{ - example: `cscli collections list -a -cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables -cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -`, - }, - installHelp: cmdHelp{ - example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, - }, - removeHelp: cmdHelp{ - example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, - }, - upgradeHelp: cmdHelp{ - example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, - }, - inspectHelp: cmdHelp{ - example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, - }, - listHelp: cmdHelp{ - example: `cscli collections list -cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables - -List only enabled collections unless "-a" or names are specified.`, - }, - }, -} - -func NewItemsCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - +func (it itemCLI) NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), @@ -247,7 +55,7 @@ func NewItemsCmd(typeName string) *cobra.Command { return cmd } -func (it hubItemType) Install(cmd *cobra.Command, args []string) error { +func (it itemCLI) Install(cmd *cobra.Command, args []string) error { flags := cmd.Flags() downloadOnly, err := flags.GetBool("download-only") @@ -295,7 +103,7 @@ func (it hubItemType) Install(cmd *cobra.Command, args []string) error { return nil } -func (it hubItemType) NewInstallCmd() *cobra.Command { +func (it itemCLI) NewInstallCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.installHelp.use, "install [item]..."), Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), @@ -330,7 +138,7 @@ func istalledParentNames(item *cwhub.Item) []string { return ret } -func (it hubItemType) Remove(cmd *cobra.Command, args []string) error { +func (it itemCLI) Remove(cmd *cobra.Command, args []string) error { flags := cmd.Flags() purge, err := flags.GetBool("purge") @@ -424,7 +232,7 @@ func (it hubItemType) Remove(cmd *cobra.Command, args []string) error { return nil } -func (it hubItemType) NewRemoveCmd() *cobra.Command { +func (it itemCLI) NewRemoveCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.removeHelp.use, "remove [item]..."), Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), @@ -446,7 +254,7 @@ func (it hubItemType) NewRemoveCmd() *cobra.Command { return cmd } -func (it hubItemType) Upgrade(cmd *cobra.Command, args []string) error { +func (it itemCLI) Upgrade(cmd *cobra.Command, args []string) error { flags := cmd.Flags() force, err := flags.GetBool("force") @@ -520,7 +328,7 @@ func (it hubItemType) Upgrade(cmd *cobra.Command, args []string) error { return nil } -func (it hubItemType) NewUpgradeCmd() *cobra.Command { +func (it itemCLI) NewUpgradeCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), @@ -540,7 +348,7 @@ func (it hubItemType) NewUpgradeCmd() *cobra.Command { return cmd } -func (it hubItemType) Inspect(cmd *cobra.Command, args []string) error { +func (it itemCLI) Inspect(cmd *cobra.Command, args []string) error { flags := cmd.Flags() url, err := flags.GetString("url") @@ -570,12 +378,18 @@ func (it hubItemType) Inspect(cmd *cobra.Command, args []string) error { if err = InspectItem(item, !noMetrics); err != nil { return err } + + if it.inspectDetail != nil { + if err = it.inspectDetail(item); err != nil { + return err + } + } } return nil } -func (it hubItemType) NewInspectCmd() *cobra.Command { +func (it itemCLI) NewInspectCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), @@ -596,7 +410,7 @@ func (it hubItemType) NewInspectCmd() *cobra.Command { return cmd } -func (it hubItemType) List(cmd *cobra.Command, args []string) error { +func (it itemCLI) List(cmd *cobra.Command, args []string) error { flags := cmd.Flags() all, err := flags.GetBool("all") @@ -623,7 +437,7 @@ func (it hubItemType) List(cmd *cobra.Command, args []string) error { return nil } -func (it hubItemType) NewListCmd() *cobra.Command { +func (it itemCLI) NewListCmd() *cobra.Command { cmd := &cobra.Command{ Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index b38d56ce95c..c9b7d7030ef 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -241,12 +241,13 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - rootCmd.AddCommand(NewAppsecRulesCmd()) // Keep it like this for now, we'll switch later to the generic implementation - rootCmd.AddCommand(NewItemsCmd("collections")) - rootCmd.AddCommand(NewItemsCmd("parsers")) - rootCmd.AddCommand(NewItemsCmd("scenarios")) - rootCmd.AddCommand(NewItemsCmd("postoverflows")) - rootCmd.AddCommand(NewItemsCmd("appsec-configs")) + + rootCmd.AddCommand(NewCollectionCLI().NewCommand()) + rootCmd.AddCommand(NewParserCLI().NewCommand()) + rootCmd.AddCommand(NewScenarioCLI().NewCommand()) + rootCmd.AddCommand(NewPostOverflowCLI().NewCommand()) + rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand()) + rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand()) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) From 1515dbda63d787aee4b8d257b6cd8ad0e6ddb7fb Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 6 Dec 2023 18:20:28 +0100 Subject: [PATCH 262/263] update to our main branch of our coraza fork --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 99454bb67a3..46bc74929fd 100644 --- a/go.mod +++ b/go.mod @@ -91,7 +91,7 @@ require ( ) require ( - github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 + github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.0 diff --git a/go.sum b/go.sum index b7ce796f18f..835b288acab 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9 h1:vFJiYtK github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 h1:vY0KZvoS4Xl9IfGucBA4l1CV1auRPPJtjZSTz/Rl6iQ= github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= From 692d96b1b503a5fa81d9b9a6048c67aa81ddb094 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Thu, 7 Dec 2023 11:54:50 +0100 Subject: [PATCH 263/263] have a better name when matching a native rule format --- pkg/acquisition/modules/appsec/utils.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index 8d879912c1a..e43313a19bf 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -225,9 +225,9 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR evt.Appsec.HasOutBandMatches = true } - name := "NOT_SET" - version := "NOT_SET" - hash := "NOT_SET" + name := "" + version := "" + hash := "" ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID()) if details, ok := appsec.AppsecRulesDetails[rule.Rule().ID()]; ok { @@ -237,6 +237,8 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR hash = details.Hash ruleNameProm = details.Name r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) + } else { + name = fmt.Sprintf("native_rule:%d", rule.Rule().ID()) } AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc()