From 00f2059e5d7bc2ca2e3e8b1562bdfede1ed570e3 Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Tue, 17 Oct 2023 09:34:30 +0700 Subject: [PATCH 01/14] docs: remove --scanners none (#5384) --- docs/docs/target/container_image.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/docs/target/container_image.md b/docs/docs/target/container_image.md index c9fd3204fc32..6b9556b8d5c5 100644 --- a/docs/docs/target/container_image.md +++ b/docs/docs/target/container_image.md @@ -113,13 +113,6 @@ You can enable it with `--image-config-scanners config`. $ trivy image --image-config-scanners config [YOUR_IMAGE_NAME] ``` -If you just want to scan the image config, you can disable scanners with `--scanners none`. -For example: - -``` -$ trivy image --scanners none --image-config-scanners config alpine:3.17.0 -``` -
Result From 9fba79f0b6b82a8cd7a74c7ccae4bc8f8edcd3b3 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Wed, 18 Oct 2023 16:21:56 +0200 Subject: [PATCH 02/14] chore(deps): move to aws-sdk-go-v2 (#5381) --- go.mod | 41 ++++++------- go.sum | 75 ++++++++++++------------ integration/aws_cloud_test.go | 6 +- pkg/fanal/cache/s3.go | 36 +++++++----- pkg/fanal/cache/s3_test.go | 40 ++++++------- pkg/fanal/image/registry/ecr/ecr.go | 51 ++++++++-------- pkg/fanal/image/registry/ecr/ecr_test.go | 16 +++-- 7 files changed, 135 insertions(+), 130 deletions(-) diff --git a/go.mod b/go.mod index 3275f1d4d3dc..9add5399c4ef 100644 --- a/go.mod +++ b/go.mod @@ -26,11 +26,14 @@ require ( github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d github.com/aquasecurity/trivy-java-db v0.0.0-20230209231723-7cddb1406728 github.com/aquasecurity/trivy-kubernetes v0.5.8-0.20230928134646-b414e546fe6d - github.com/aws/aws-sdk-go v1.45.19 - github.com/aws/aws-sdk-go-v2 v1.21.0 - github.com/aws/aws-sdk-go-v2/config v1.18.38 + github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2/config v1.18.45 + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 github.com/aws/aws-sdk-go-v2/service/ec2 v1.98.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 + github.com/aws/aws-sdk-go-v2/service/ecr v1.17.18 + github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cheggaaa/pb/v3 v3.1.4 @@ -146,13 +149,13 @@ require ( github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.36 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect + github.com/aws/aws-sdk-go v1.45.19 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.16.0 // indirect github.com/aws/aws-sdk-go-v2/service/apigateway v1.15.24 // indirect github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.13.11 // indirect @@ -165,7 +168,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/docdb v1.19.11 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.7 // indirect github.com/aws/aws-sdk-go-v2/service/ebs v1.18.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ecr v1.17.18 // indirect github.com/aws/aws-sdk-go-v2/service/ecs v1.28.1 // indirect github.com/aws/aws-sdk-go-v2/service/efs v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/service/eks v1.27.14 // indirect @@ -174,11 +176,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/elasticsearchservice v1.19.0 // indirect github.com/aws/aws-sdk-go-v2/service/emr v1.24.4 // indirect github.com/aws/aws-sdk-go-v2/service/iam v1.21.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.23 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect github.com/aws/aws-sdk-go-v2/service/kafka v1.19.4 // indirect github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.19 // indirect github.com/aws/aws-sdk-go-v2/service/kms v1.24.1 // indirect @@ -187,14 +189,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/neptune v1.20.7 // indirect github.com/aws/aws-sdk-go-v2/service/rds v1.26.1 // indirect github.com/aws/aws-sdk-go-v2/service/redshift v1.27.7 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 // indirect github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.2 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.20.10 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect github.com/aws/aws-sdk-go-v2/service/workspaces v1.23.0 // indirect - github.com/aws/smithy-go v1.14.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/briandowns/spinner v1.23.0 // indirect diff --git a/go.sum b/go.sum index a3c509514a3f..9e972ed49296 100644 --- a/go.sum +++ b/go.sum @@ -374,17 +374,19 @@ github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3eP github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= -github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.38 h1:CByQCELMgm2tM1lAehx3XNg0R/pfeXsYzqn0Aq2chJQ= -github.com/aws/aws-sdk-go-v2/config v1.18.38/go.mod h1:vNm9Hf5VgG2fSUWhT3zFrqN/RosGcabFMYgiSoxKFU8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.36 h1:ps0cPswZjpsOk6sLwG6fdXTzrYjCplgPEyG3OUbbdqE= -github.com/aws/aws-sdk-go-v2/credentials v1.13.36/go.mod h1:sY2phUzxbygoyDtTXhqi7GjGjCQ1S5a5Rj8u3ksBxCg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90 h1:mtJRt80k1oGw7QQPluAx8AZ6u16MyCA2di/lMhagZ7I= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.90/go.mod h1:lYwZTkeMQWPvNU+u7oYArdNhQ8EKiSGU76jVv0w2GH4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= @@ -394,8 +396,8 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= @@ -405,12 +407,12 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 h1:AzwRi5OKKwo4QNqPf7TjeO+tK8AyOK3GVSwmRPo7/Cs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6/go.mod h1:Q0Hq2X/NuL7z8b1Dww8rmOFl+jzusKEcyvkKspwdpyc= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.16.0 h1:rPv8ZiaTIwLp4JOCQAQcgPx7i2a7FTRY7lnyrNS0HbU= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.16.0/go.mod h1:l5+hat25VFsG9jpsXrtEYqw6Ih3pLaC5I4+8hrng7F4= github.com/aws/aws-sdk-go-v2/service/apigateway v1.15.24 h1:eWwaF3m67oAJGBhfzVC9dLXPYhLHB238N1LhgFa8INk= @@ -456,20 +458,20 @@ github.com/aws/aws-sdk-go-v2/service/emr v1.24.4/go.mod h1:hvWrBVsomnNf7Y0Onrl+w github.com/aws/aws-sdk-go-v2/service/iam v1.21.1 h1:VTCWgsrromZqnlRgfziqqWWcW7LFkQLwJVYgf/5zgWA= github.com/aws/aws-sdk-go-v2/service/iam v1.21.1/go.mod h1:LBsjrFczXiQLASO6FtDGTeHuZh6oHuIH6VKaOozFghg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 h1:vGWm5vTpMr39tEZfQeDiDAMgk+5qsnvRny3FjLpnH5w= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19/go.mod h1:2WpVWFC5n4DYhjNXzObtge8xfgId9UP6GWca46KJFLo= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.23 h1:5AwQnYQT3ZX/N7hPTAx4ClWyucaiqr2esQRMNbJIby0= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.23/go.mod h1:s8OUYECPoPpevQHmRmMBemFIx6Oc91iapsw56KiXIMY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 h1:NbWkRxEEIRSCqxhsHQuMiTH7yo+JZW1gp8v3elSVMTQ= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= github.com/aws/aws-sdk-go-v2/service/kafka v1.19.4 h1:EmEk3gRJzSEK8DUnkvTGDn/RQc/Ud+yimUFIG+PSvTI= github.com/aws/aws-sdk-go-v2/service/kafka v1.19.4/go.mod h1:+O9qi0UC83Lk0KAnC/ixNcw4piXfUtPzXpYn/KC2Mhg= github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.19 h1:qVaBkJxFxm6o/9DPNnJU6L9O3V7ycEKhCvRm2BFBQTU= @@ -486,29 +488,28 @@ github.com/aws/aws-sdk-go-v2/service/rds v1.26.1 h1:tiXsw36GaRUWMcH5uRM2uM7vo+bN github.com/aws/aws-sdk-go-v2/service/rds v1.26.1/go.mod h1:d8jJiNpy2cyl52sw5msQQ12ajEbPAK+twYPR7J35slw= github.com/aws/aws-sdk-go-v2/service/redshift v1.27.7 h1:fKg773iDMTGUxd8UNkEfwYGNjT6H6KFSmqV97Yte+jc= github.com/aws/aws-sdk-go-v2/service/redshift v1.27.7/go.mod h1:jLAH4E3fjUxkBhu7vcx7eCSurnq7q1qMyAB1VZvvbAk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 h1:Ll5/YVCOzRB+gxPqs2uD0R7/MyATC0w85626glSKmp4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzKgaq7TcxzmEoDWN8lM0qzQ= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.2 h1:3x1Qilin49XQ1rK6pDNAfG+DmCFPfB7Rrpl+FUDAR/0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.2/go.mod h1:HEBBc70BYi5eUvxBqC3xXjU/04NO96X/XNUe5qhC7Bc= github.com/aws/aws-sdk-go-v2/service/sns v1.20.10 h1:pJ/iXyg9aD5Hg2FRHQjrWPDyabsP6R3aqxaXqscAVKk= github.com/aws/aws-sdk-go-v2/service/sns v1.20.10/go.mod h1:WjBcrd28zNbbuAcIRO/n89sSeOxTuOZPiuxNXU/2WrI= github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 h1:4P/vyx7zCI5yBhlDZ2kwhoLjMJi0X7iR3cxqjNfbego= github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6/go.mod h1:HQHh1eChX10zDnGmD53WLYk8nPhUKO/JkAUUzDZ530Y= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= -github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk= -github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= github.com/aws/aws-sdk-go-v2/service/workspaces v1.23.0 h1:lrgZ9pZm9utPOPAXmQhqtf8oWRRksoSFxOE8RoD+pHc= github.com/aws/aws-sdk-go-v2/service/workspaces v1.23.0/go.mod h1:vPam8+zGthTXeaFWgl3Uqbzo/0QEoXF22jpuMZ97hSk= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= -github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= diff --git a/integration/aws_cloud_test.go b/integration/aws_cloud_test.go index 53b47ee345e9..53bcecf5220f 100644 --- a/integration/aws_cloud_test.go +++ b/integration/aws_cloud_test.go @@ -36,13 +36,16 @@ func TestAwsCommandRun(t *testing.T) { }, { name: "fail without creds", + envs: map[string]string{ + "AWS_PROFILE": "non-existent-profile", + }, options: flag.Options{ RegoOptions: flag.RegoOptions{SkipPolicyUpdate: true}, AWSOptions: flag.AWSOptions{ Region: "us-east-1", }, }, - wantErr: "failed to retrieve credentials", + wantErr: "non-existent-profile", }, } @@ -57,7 +60,6 @@ func TestAwsCommandRun(t *testing.T) { tt.options.AWSOptions.Endpoint = addr tt.options.GlobalOptions.Timeout = time.Minute - t.Setenv("AWS_PROFILE", "non-existent-profile") for k, v := range tt.envs { t.Setenv(k, v) } diff --git a/pkg/fanal/cache/s3.go b/pkg/fanal/cache/s3.go index 648f7aa7df03..b11aed42a476 100644 --- a/pkg/fanal/cache/s3.go +++ b/pkg/fanal/cache/s3.go @@ -2,13 +2,13 @@ package cache import ( "bytes" + "context" "encoding/json" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" @@ -17,14 +17,20 @@ import ( var _ Cache = &S3Cache{} +type s3API interface { + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + DeleteBucket(ctx context.Context, params *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) +} + type S3Cache struct { - s3Client s3iface.S3API - downloader s3manageriface.DownloaderAPI + s3Client s3API + downloader *manager.Downloader bucketName string prefix string } -func NewS3Cache(bucketName, prefix string, api s3iface.S3API, downloaderAPI s3manageriface.DownloaderAPI) S3Cache { +func NewS3Cache(bucketName, prefix string, api s3API, downloaderAPI *manager.Downloader) S3Cache { return S3Cache{ s3Client: api, downloader: downloaderAPI, @@ -46,7 +52,7 @@ func (c S3Cache) DeleteBlobs(blobIDs []string) error { for _, blobID := range blobIDs { key := fmt.Sprintf("%s/%s/%s", blobBucket, c.prefix, blobID) input := &s3.DeleteBucketInput{Bucket: aws.String(key)} - if _, err := c.s3Client.DeleteBucket(input); err != nil { + if _, err := c.s3Client.DeleteBucket(context.TODO(), input); err != nil { errs = multierror.Append(errs, err) } } @@ -71,12 +77,12 @@ func (c S3Cache) put(key string, body interface{}) (err error) { Key: aws.String(key), Body: bytes.NewReader(b), } - _, err = c.s3Client.PutObject(params) + _, err = c.s3Client.PutObject(context.TODO(), params) if err != nil { return xerrors.Errorf("unable to put object: %w", err) } // Index file due S3 caveat read after write consistency - _, err = c.s3Client.PutObject(&s3.PutObjectInput{ + _, err = c.s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(c.bucketName), Key: aws.String(fmt.Sprintf("%s.index", key)), }) @@ -88,8 +94,8 @@ func (c S3Cache) put(key string, body interface{}) (err error) { func (c S3Cache) GetBlob(blobID string) (types.BlobInfo, error) { var blobInfo types.BlobInfo - buf := aws.NewWriteAtBuffer([]byte{}) - _, err := c.downloader.Download(buf, &s3.GetObjectInput{ + buf := manager.NewWriteAtBuffer([]byte{}) + _, err := c.downloader.Download(context.TODO(), buf, &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), Key: aws.String(fmt.Sprintf("%s/%s/%s", blobBucket, c.prefix, blobID)), }) @@ -105,8 +111,8 @@ func (c S3Cache) GetBlob(blobID string) (types.BlobInfo, error) { func (c S3Cache) GetArtifact(artifactID string) (types.ArtifactInfo, error) { var info types.ArtifactInfo - buf := aws.NewWriteAtBuffer([]byte{}) - _, err := c.downloader.Download(buf, &s3.GetObjectInput{ + buf := manager.NewWriteAtBuffer([]byte{}) + _, err := c.downloader.Download(context.TODO(), buf, &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), Key: aws.String(fmt.Sprintf("%s/%s/%s", artifactBucket, c.prefix, artifactID)), }) @@ -121,7 +127,7 @@ func (c S3Cache) GetArtifact(artifactID string) (types.ArtifactInfo, error) { } func (c S3Cache) getIndex(key, keyType string) error { - _, err := c.s3Client.HeadObject(&s3.HeadObjectInput{ + _, err := c.s3Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Key: aws.String(fmt.Sprintf("%s/%s/%s.index", keyType, c.prefix, key)), Bucket: &c.bucketName, }) diff --git a/pkg/fanal/cache/s3_test.go b/pkg/fanal/cache/s3_test.go index cf96a3e0b716..ed3da27b974f 100644 --- a/pkg/fanal/cache/s3_test.go +++ b/pkg/fanal/cache/s3_test.go @@ -1,36 +1,36 @@ package cache import ( + "context" "errors" "reflect" "testing" "time" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/types" ) type mockS3Client struct { - s3iface.S3API + s3API } const ( correctHash = "sha256:24df0d4e20c0f42d3703bf1f1db2bdd77346c7956f74f423603d651e8e5ae8a7" ) -func (m *mockS3Client) PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) { +func (m *mockS3Client) PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { return &s3.PutObjectOutput{}, nil } -func (m *mockS3Client) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { +func (m *mockS3Client) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{}, nil } -func (m *mockS3Client) DeleteBucket(in *s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error) { +func (m *mockS3Client) DeleteBucket(ctx context.Context, in *s3.DeleteBucketInput, optFns ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) { if in != nil && *in.Bucket == blobBucket+"/prefix/"+correctHash { return &s3.DeleteBucketOutput{}, nil } @@ -41,8 +41,8 @@ func TestS3Cache_PutBlob(t *testing.T) { mockSvc := &mockS3Client{} type fields struct { - S3 s3iface.S3API - Downloader *s3manager.Downloader + S3 s3API + Downloader *manager.Downloader BucketName string Prefix string } @@ -88,8 +88,8 @@ func TestS3Cache_PutArtifact(t *testing.T) { mockSvc := &mockS3Client{} type fields struct { - S3 s3iface.S3API - Downloader *s3manager.Downloader + S3 s3API + Downloader *manager.Downloader BucketName string Prefix string } @@ -141,8 +141,8 @@ func TestS3Cache_getIndex(t *testing.T) { mockSvc := &mockS3Client{} type fields struct { - S3 s3iface.S3API - Downloader *s3manager.Downloader + S3 s3API + Downloader *manager.Downloader BucketName string Prefix string } @@ -181,14 +181,14 @@ func TestS3Cache_getIndex(t *testing.T) { } type mockS3ClientMissingBlobs struct { - s3iface.S3API + s3API } -func (m *mockS3ClientMissingBlobs) PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) { +func (m *mockS3ClientMissingBlobs) PutObject(ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { return &s3.PutObjectOutput{}, nil } -func (m *mockS3ClientMissingBlobs) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { +func (m *mockS3ClientMissingBlobs) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{}, xerrors.Errorf("the object doesn't exist in S3") } @@ -196,8 +196,8 @@ func TestS3Cache_MissingBlobs(t *testing.T) { mockSvc := &mockS3ClientMissingBlobs{} type fields struct { - S3 s3iface.S3API - Downloader *s3manager.Downloader + S3 s3API + Downloader *manager.Downloader BucketName string Prefix string } @@ -252,8 +252,8 @@ func TestS3Cache_DeleteBlobs(t *testing.T) { mockSvc := &mockS3Client{} type fields struct { - S3 s3iface.S3API - Downloader *s3manager.Downloader + S3 s3API + Downloader *manager.Downloader BucketName string Prefix string } diff --git a/pkg/fanal/image/registry/ecr/ecr.go b/pkg/fanal/image/registry/ecr/ecr.go index 72d5204d0375..e675ed47afaf 100644 --- a/pkg/fanal/image/registry/ecr/ecr.go +++ b/pkg/fanal/image/registry/ecr/ecr.go @@ -5,11 +5,10 @@ import ( "encoding/base64" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ecr" - "github.com/aws/aws-sdk-go/service/ecr/ecriface" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/ecr" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/types" @@ -17,46 +16,44 @@ import ( const ecrURL = "amazonaws.com" +type ecrAPI interface { + GetAuthorizationToken(ctx context.Context, params *ecr.GetAuthorizationTokenInput, optFns ...func(*ecr.Options)) (*ecr.GetAuthorizationTokenOutput, error) +} + type ECR struct { - Client ecriface.ECRAPI + Client ecrAPI } -func getSession(option types.RegistryOptions) (*session.Session, error) { +func getSession(option types.RegistryOptions) (aws.Config, error) { // create custom credential information if option is valid if option.AWSSecretKey != "" && option.AWSAccessKey != "" && option.AWSRegion != "" { - return session.NewSessionWithOptions( - session.Options{ - Config: aws.Config{ - Region: aws.String(option.AWSRegion), - Credentials: credentials.NewStaticCredentialsFromCreds( - credentials.Value{ - AccessKeyID: option.AWSAccessKey, - SecretAccessKey: option.AWSSecretKey, - SessionToken: option.AWSSessionToken, - }, - ), - }, - }) + return config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(option.AWSRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(option.AWSAccessKey, option.AWSSecretKey, option.AWSSessionToken)), + ) } - // use shared configuration normally - return session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - }) + return config.LoadDefaultConfig(context.TODO()) } func (e *ECR) CheckOptions(domain string, option types.RegistryOptions) error { if !strings.HasSuffix(domain, ecrURL) { return xerrors.Errorf("ECR : %w", types.InvalidURLPattern) } - sess := session.Must(getSession(option)) - svc := ecr.New(sess) + + cfg, err := getSession(option) + if err != nil { + return err + } + + svc := ecr.NewFromConfig(cfg) e.Client = svc return nil } func (e *ECR) GetCredential(ctx context.Context) (username, password string, err error) { input := &ecr.GetAuthorizationTokenInput{} - result, err := e.Client.GetAuthorizationTokenWithContext(ctx, input) + result, err := e.Client.GetAuthorizationToken(ctx, input) if err != nil { return "", "", xerrors.Errorf("failed to get authorization token: %w", err) } diff --git a/pkg/fanal/image/registry/ecr/ecr_test.go b/pkg/fanal/image/registry/ecr/ecr_test.go index b55f56d7b179..63ae1858114c 100644 --- a/pkg/fanal/image/registry/ecr/ecr_test.go +++ b/pkg/fanal/image/registry/ecr/ecr_test.go @@ -5,10 +5,9 @@ import ( "errors" "testing" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/ecr" - "github.com/aws/aws-sdk-go/service/ecr/ecriface" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ecr" + awstypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" "github.com/aquasecurity/trivy/pkg/fanal/types" ) @@ -40,11 +39,10 @@ func TestCheckOptions(t *testing.T) { } type mockedECR struct { - ecriface.ECRAPI Resp ecr.GetAuthorizationTokenOutput } -func (m mockedECR) GetAuthorizationTokenWithContext(ctx context.Context, input *ecr.GetAuthorizationTokenInput, options ...request.Option) (*ecr.GetAuthorizationTokenOutput, error) { +func (m mockedECR) GetAuthorizationToken(ctx context.Context, params *ecr.GetAuthorizationTokenInput, optFns ...func(*ecr.Options)) (*ecr.GetAuthorizationTokenOutput, error) { return &m.Resp, nil } @@ -56,7 +54,7 @@ func TestECRGetCredential(t *testing.T) { }{ { Resp: ecr.GetAuthorizationTokenOutput{ - AuthorizationData: []*ecr.AuthorizationData{ + AuthorizationData: []awstypes.AuthorizationData{ {AuthorizationToken: aws.String("YXdzOnBhc3N3b3Jk")}, }, }, @@ -65,7 +63,7 @@ func TestECRGetCredential(t *testing.T) { }, { Resp: ecr.GetAuthorizationTokenOutput{ - AuthorizationData: []*ecr.AuthorizationData{ + AuthorizationData: []awstypes.AuthorizationData{ {AuthorizationToken: aws.String("YXdzOnBhc3N3b3JkOmJhZA==")}, }, }, @@ -74,7 +72,7 @@ func TestECRGetCredential(t *testing.T) { }, { Resp: ecr.GetAuthorizationTokenOutput{ - AuthorizationData: []*ecr.AuthorizationData{ + AuthorizationData: []awstypes.AuthorizationData{ {AuthorizationToken: aws.String("YXdzcGFzc3dvcmQ=")}, }, }, From e5317c7bc261081ae4711c4f68e6704462c6a269 Mon Sep 17 00:00:00 2001 From: Erick Redwine Date: Wed, 18 Oct 2023 21:29:54 -0600 Subject: [PATCH 03/14] fix: correct invalid MD5 hashes for rpms ending with one or more zero bytes (#5393) --- go.mod | 2 +- go.sum | 4 ++-- pkg/fanal/analyzer/pkg/rpm/rpm_test.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9add5399c4ef..5d496371fa04 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 - github.com/knqyf263/go-rpmdb v0.0.0-20230517124904-b97c85e63254 + github.com/knqyf263/go-rpmdb v0.0.0-20231008124120-ac49267ab4e1 github.com/knqyf263/nested v0.0.1 github.com/kylelemons/godebug v1.1.0 github.com/magefile/mage v1.15.0 diff --git a/go.sum b/go.sum index 9e972ed49296..1fdfe563f603 100644 --- a/go.sum +++ b/go.sum @@ -1217,8 +1217,8 @@ github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 h1:PPPlUUq github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422/go.mod h1:ijAmSS4jErO6+KRzcK6ixsm3Vt96hMhJ+W+x+VmbrQA= github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 h1:aC6MEAs3PE3lWD7lqrJfDxHd6hcced9R4JTZu85cJwU= github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075/go.mod h1:i4sF0l1fFnY1aiw08QQSwVAFxHEm311Me3WsU/X7nL0= -github.com/knqyf263/go-rpmdb v0.0.0-20230517124904-b97c85e63254 h1:sWdRcroy0oENkvpa1jH7g8KyN7jBFsooMNlFrAvpI5I= -github.com/knqyf263/go-rpmdb v0.0.0-20230517124904-b97c85e63254/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww= +github.com/knqyf263/go-rpmdb v0.0.0-20231008124120-ac49267ab4e1 h1:lrciwn7tj0j7HS5DfpAFnFZEqxzPGIkVOVS89dLOkf0= +github.com/knqyf263/go-rpmdb v0.0.0-20231008124120-ac49267ab4e1/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww= github.com/knqyf263/nested v0.0.1 h1:Sv26CegUMhjt19zqbBKntjwESdxe5hxVPSk0+AKjdUc= github.com/knqyf263/nested v0.0.1/go.mod h1:zwhsIhMkBg90DTOJQvxPkKIypEHPYkgWHs4gybdlUmk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/pkg/fanal/analyzer/pkg/rpm/rpm_test.go b/pkg/fanal/analyzer/pkg/rpm/rpm_test.go index 1b19e4ceedde..9cb29cbb09b6 100644 --- a/pkg/fanal/analyzer/pkg/rpm/rpm_test.go +++ b/pkg/fanal/analyzer/pkg/rpm/rpm_test.go @@ -74,7 +74,7 @@ func TestParseRpmInfo(t *testing.T) { {Name: "libusbx", Epoch: 0, Version: "1.0.22", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:0a5650b09182d2b0134ee832acd5432e", SrcName: "libusbx", SrcEpoch: 0, SrcVersion: "1.0.22", SrcRelease: "1.fc28"}, {Name: "libsemanage", Epoch: 0, Version: "2.8", Release: "2.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:5c29e3e9ed089e3fedf7298dabe503fd", SrcName: "libsemanage", SrcEpoch: 0, SrcVersion: "2.8", SrcRelease: "2.fc28"}, {Name: "libutempter", Epoch: 0, Version: "1.1.6", Release: "14.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:ccfd737faad3d1be60f4c21839f5a9af", SrcName: "libutempter", SrcEpoch: 0, SrcVersion: "1.1.6", SrcRelease: "14.fc28"}, - {Name: "mpfr", Epoch: 0, Version: "3.1.6", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv3+ and GPLv3+ and GFDL"}, Maintainer: "Fedora Project", Digest: "md5:a1339297aae3e31e8e1b1de00f7930", SrcName: "mpfr", SrcEpoch: 0, SrcVersion: "3.1.6", SrcRelease: "1.fc28"}, + {Name: "mpfr", Epoch: 0, Version: "3.1.6", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv3+ and GPLv3+ and GFDL"}, Maintainer: "Fedora Project", Digest: "md5:a1339297aae3e31e8e1b1de00f793000", SrcName: "mpfr", SrcEpoch: 0, SrcVersion: "3.1.6", SrcRelease: "1.fc28"}, {Name: "gnutls", Epoch: 0, Version: "3.6.3", Release: "4.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"GPLv3+ and LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:6c813ffeecd51d2298766d86859395ca", SrcName: "gnutls", SrcEpoch: 0, SrcVersion: "3.6.3", SrcRelease: "4.fc28"}, {Name: "gzip", Epoch: 0, Version: "1.9", Release: "3.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"GPLv3+ and GFDL"}, Maintainer: "Fedora Project", Digest: "md5:efc5209278a36c1b62ee770a95e59929", SrcName: "gzip", SrcEpoch: 0, SrcVersion: "1.9", SrcRelease: "3.fc28"}, {Name: "acl", Epoch: 0, Version: "2.2.53", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"GPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:9f72a1f855b12dc038e3f4921c50db92", SrcName: "acl", SrcEpoch: 0, SrcVersion: "2.2.53", SrcRelease: "1.fc28"}, @@ -96,8 +96,8 @@ func TestParseRpmInfo(t *testing.T) { {Name: "openldap", Epoch: 0, Version: "2.4.46", Release: "3.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"OpenLDAP"}, Maintainer: "Fedora Project", Digest: "md5:c98aaadc1c628656c7f90a18759da517", SrcName: "openldap", SrcEpoch: 0, SrcVersion: "2.4.46", SrcRelease: "3.fc28"}, {Name: "libseccomp", Epoch: 0, Version: "2.3.3", Release: "2.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2"}, Maintainer: "Fedora Project", Digest: "md5:26c6a917063d5654083974de6734112d", SrcName: "libseccomp", SrcEpoch: 0, SrcVersion: "2.3.3", SrcRelease: "2.fc28"}, {Name: "npth", Epoch: 0, Version: "1.5", Release: "4.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:026afa87b50e9ec9812501188b6fd998", SrcName: "npth", SrcEpoch: 0, SrcVersion: "1.5", SrcRelease: "4.fc28"}, - {Name: "gpgme", Epoch: 0, Version: "1.10.0", Release: "4.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:5a044754d91ca7d0b4e4428b052de6", SrcName: "gpgme", SrcEpoch: 0, SrcVersion: "1.10.0", SrcRelease: "4.fc28"}, - {Name: "json-c", Epoch: 0, Version: "0.13.1", Release: "2.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"MIT"}, Maintainer: "Fedora Project", Digest: "md5:2d0c0ea344c78a2faeab869c9474f9", SrcName: "json-c", SrcEpoch: 0, SrcVersion: "0.13.1", SrcRelease: "2.fc28"}, + {Name: "gpgme", Epoch: 0, Version: "1.10.0", Release: "4.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"LGPLv2+"}, Maintainer: "Fedora Project", Digest: "md5:5a044754d91ca7d0b4e4428b052de600", SrcName: "gpgme", SrcEpoch: 0, SrcVersion: "1.10.0", SrcRelease: "4.fc28"}, + {Name: "json-c", Epoch: 0, Version: "0.13.1", Release: "2.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"MIT"}, Maintainer: "Fedora Project", Digest: "md5:2d0c0ea344c78a2faeab869c9474f900", SrcName: "json-c", SrcEpoch: 0, SrcVersion: "0.13.1", SrcRelease: "2.fc28"}, {Name: "libyaml", Epoch: 0, Version: "0.1.7", Release: "5.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"MIT"}, Maintainer: "Fedora Project", Digest: "md5:4e0f94eba782aa090fe6487142772aba", SrcName: "libyaml", SrcEpoch: 0, SrcVersion: "0.1.7", SrcRelease: "5.fc28"}, {Name: "libpkgconf", Epoch: 0, Version: "1.4.2", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"ISC"}, Maintainer: "Fedora Project", Digest: "md5:5721f7af98fca6a80db3d51fe0ad7312", SrcName: "pkgconf", SrcEpoch: 0, SrcVersion: "1.4.2", SrcRelease: "1.fc28"}, {Name: "pkgconf-pkg-config", Epoch: 0, Version: "1.4.2", Release: "1.fc28", Arch: "x86_64", Modularitylabel: "", Licenses: []string{"ISC"}, Maintainer: "Fedora Project", Digest: "md5:f297678d95f451aa7132d15aff3246a7", SrcName: "pkgconf", SrcEpoch: 0, SrcVersion: "1.4.2", SrcRelease: "1.fc28"}, From 6040d9f43a35819d4c0338839b97619338e753ed Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:37:32 +0600 Subject: [PATCH 04/14] fix(sbom): add missed `primaryURL` and `source severity` for CycloneDX (#5399) Signed-off-by: knqyf263 Co-authored-by: knqyf263 --- integration/testdata/pom-cyclonedx.json.golden | 6 ++++++ pkg/sbom/cyclonedx/core/cyclonedx.go | 15 +++++++-------- pkg/sbom/cyclonedx/marshal_test.go | 11 +++++++++++ pkg/vulnerability/vulnerability.go | 7 +++++++ pkg/vulnerability/vulnerability_test.go | 9 ++++++--- 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/integration/testdata/pom-cyclonedx.json.golden b/integration/testdata/pom-cyclonedx.json.golden index 8e1c482e0580..3e87c28beedd 100644 --- a/integration/testdata/pom-cyclonedx.json.golden +++ b/integration/testdata/pom-cyclonedx.json.golden @@ -149,6 +149,9 @@ "description": "FasterXML jackson-databind 2.x before 2.9.10.4 mishandles the interaction between serialization gadgets and typing, related to br.com.anteros.dbcp.AnterosDBCPConfig (aka anteros-core).", "recommendation": "Upgrade com.fasterxml.jackson.core:jackson-databind to version 2.9.10.4", "advisories": [ + { + "url": "https://avd.aquasec.com/nvd/cve-2020-9548" + }, { "url": "https://access.redhat.com/security/cve/CVE-2020-9548" }, @@ -268,6 +271,9 @@ "description": "A flaw was found in jackson-databind before 2.9.10.7. FasterXML mishandles the interaction between serialization gadgets and typing. The highest threat from this vulnerability is to data confidentiality and integrity as well as system availability.", "recommendation": "Upgrade com.fasterxml.jackson.core:jackson-databind to version 2.9.10.7", "advisories": [ + { + "url": "https://avd.aquasec.com/nvd/cve-2021-20190" + }, { "url": "https://access.redhat.com/security/cve/CVE-2021-20190" }, diff --git a/pkg/sbom/cyclonedx/core/cyclonedx.go b/pkg/sbom/cyclonedx/core/cyclonedx.go index 9a2a7a14466f..dad245b5239c 100644 --- a/pkg/sbom/cyclonedx/core/cyclonedx.go +++ b/pkg/sbom/cyclonedx/core/cyclonedx.go @@ -155,7 +155,7 @@ func (c *CycloneDX) marshalVulnerability(bomRef string, vuln types.DetectedVulne Ratings: cdxRatings(vuln), CWEs: cwes(vuln.CweIDs), Description: vuln.Description, - Advisories: cdxAdvisories(vuln.References), + Advisories: cdxAdvisories(append([]string{vuln.PrimaryURL}, vuln.References...)), } if vuln.FixedVersion != "" { v.Recommendation = fmt.Sprintf("Upgrade %s to version %s", vuln.PkgName, vuln.FixedVersion) @@ -341,19 +341,18 @@ func UnmarshalProperties(properties *[]cdx.Property) map[string]string { } func cdxAdvisories(refs []string) *[]cdx.Advisory { + refs = lo.Uniq(refs) + advs := lo.FilterMap(refs, func(ref string, _ int) (cdx.Advisory, bool) { + return cdx.Advisory{URL: ref}, ref != "" + }) + // cyclonedx converts link to empty `[]cdx.Advisory` to `null` // `bom-1.5.schema.json` doesn't support this - `Invalid type. Expected: array, given: null` // we need to explicitly set `nil` for empty `refs` slice - if len(refs) == 0 { + if len(advs) == 0 { return nil } - var advs []cdx.Advisory - for _, ref := range refs { - advs = append(advs, cdx.Advisory{ - URL: ref, - }) - } return &advs } diff --git a/pkg/sbom/cyclonedx/marshal_test.go b/pkg/sbom/cyclonedx/marshal_test.go index de41d837f453..ddc68131ea0c 100644 --- a/pkg/sbom/cyclonedx/marshal_test.go +++ b/pkg/sbom/cyclonedx/marshal_test.go @@ -522,6 +522,11 @@ func TestMarshaler_Marshal(t *testing.T) { Description: "In GNU Binutils 2.31.1, there is a use-after-free in the error function in elfcomm.c when called from the process_archive function in readelf.c via a crafted ELF file.", Published: "2018-12-31T19:29:00+00:00", Updated: "2019-10-31T01:15:00+00:00", + Advisories: &[]cdx.Advisory{ + { + URL: "https://avd.aquasec.com/nvd/cve-2018-20623", + }, + }, Affects: &[]cdx.Affects{ { Ref: "pkg:rpm/centos/binutils@2.30-93.el8?arch=aarch64&distro=centos-8.3.2011", @@ -991,6 +996,9 @@ func TestMarshaler_Marshal(t *testing.T) { }, Description: "Action Pack is a framework for handling and responding to web requests. Under certain circumstances response bodies will not be closed. In the event a response is *not* notified of a `close`, `ActionDispatch::Executor` will not know to reset thread local state for the next request. This can lead to data being leaked to subsequent requests.This has been fixed in Rails 7.0.2.1, 6.1.4.5, 6.0.4.5, and 5.2.6.1. Upgrading is highly recommended, but to work around this problem a middleware described in GHSA-wh98-p28r-vrc9 can be used.", Advisories: &[]cdx.Advisory{ + { + URL: "https://avd.aquasec.com/nvd/cve-2022-23633", + }, { URL: "http://www.openwall.com/lists/oss-security/2022/02/11/5", }, @@ -1384,6 +1392,9 @@ func TestMarshaler_Marshal(t *testing.T) { CWEs: lo.ToPtr([]int{94}), Description: "The DBCPConnectionPool and HikariCPConnectionPool Controller Services in Apache NiFi 0.0.2 through 1.21.0...", Advisories: &[]cdx.Advisory{ + { + URL: "https://avd.aquasec.com/nvd/cve-2023-34468", + }, { URL: "http://www.openwall.com/lists/oss-security/2023/06/12/3", }, diff --git a/pkg/vulnerability/vulnerability.go b/pkg/vulnerability/vulnerability.go index 2b29e632e0f6..504d9293e873 100644 --- a/pkg/vulnerability/vulnerability.go +++ b/pkg/vulnerability/vulnerability.go @@ -91,6 +91,13 @@ func (c Client) FillInfo(vulns []types.DetectedVulnerability) { if vulns[i].SeveritySource != "" { severity = vulns[i].Severity severitySource = vulns[i].SeveritySource + + // Store package-specific severity in vendor severities + if vuln.VendorSeverity == nil { + vuln.VendorSeverity = make(dbTypes.VendorSeverity) + } + s, _ := dbTypes.NewSeverity(severity) // skip error handling because `SeverityUnknown` will be returned in case of error + vuln.VendorSeverity[severitySource] = s } // Add the vulnerability detail diff --git a/pkg/vulnerability/vulnerability_test.go b/pkg/vulnerability/vulnerability_test.go index 8caf314ae7fa..11dac691503e 100644 --- a/pkg/vulnerability/vulnerability_test.go +++ b/pkg/vulnerability/vulnerability_test.go @@ -218,9 +218,12 @@ func TestClient_FillInfo(t *testing.T) { Status: dbTypes.StatusAffected, SeveritySource: vulnerability.Debian, Vulnerability: dbTypes.Vulnerability{ - Title: "dos", - Description: "dos vulnerability", - Severity: dbTypes.SeverityLow.String(), + Title: "dos", + Description: "dos vulnerability", + Severity: dbTypes.SeverityLow.String(), + VendorSeverity: map[dbTypes.SourceID]dbTypes.Severity{ + vulnerability.Debian: dbTypes.SeverityLow, + }, References: []string{"http://example.com"}, LastModifiedDate: utils.MustTimeParse("2020-01-01T01:01:00Z"), PublishedDate: utils.MustTimeParse("2001-01-01T01:01:00Z"), From f2a12f5f90f940effe5dfc763025a70b127ec539 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Fri, 20 Oct 2023 10:43:15 +0900 Subject: [PATCH 05/14] refactor: define a new struct for scan targets (#5397) --- pkg/fanal/types/artifact.go | 2 +- pkg/scanner/langpkg/scan.go | 35 ++++---- pkg/scanner/local/scan.go | 174 +++++++++++++++++++++++------------- pkg/scanner/ospkg/scan.go | 58 ++++-------- pkg/types/scan.go | 33 +++++++ pkg/types/scanoptions.go | 17 ---- 6 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 pkg/types/scan.go delete mode 100644 pkg/types/scanoptions.go diff --git a/pkg/fanal/types/artifact.go b/pkg/fanal/types/artifact.go index c60d68fc75e6..7db2cdbf1949 100644 --- a/pkg/fanal/types/artifact.go +++ b/pkg/fanal/types/artifact.go @@ -264,7 +264,7 @@ type BlobInfo struct { CustomResources []CustomResource `json:",omitempty"` } -// ArtifactDetail is generated by applying blobs +// ArtifactDetail represents the analysis result. type ArtifactDetail struct { OS OS `json:",omitempty"` Repository *Repository `json:",omitempty"` diff --git a/pkg/scanner/langpkg/scan.go b/pkg/scanner/langpkg/scan.go index df16c0c78476..9480c8a13614 100644 --- a/pkg/scanner/langpkg/scan.go +++ b/pkg/scanner/langpkg/scan.go @@ -23,8 +23,8 @@ var ( ) type Scanner interface { - Packages(detail ftypes.ArtifactDetail, options types.ScanOptions) types.Results - Scan(detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Results, error) + Packages(target types.ScanTarget, options types.ScanOptions) types.Results + Scan(target types.ScanTarget, options types.ScanOptions) (types.Results, error) } type scanner struct{} @@ -33,20 +33,15 @@ func NewScanner() Scanner { return &scanner{} } -func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) types.Results { +func (s *scanner) Packages(target types.ScanTarget, _ types.ScanOptions) types.Results { var results types.Results - for _, app := range detail.Applications { + for _, app := range target.Applications { if len(app.Libraries) == 0 { continue } - target := app.FilePath - if t, ok := PkgTargets[app.Type]; ok && target == "" { - // When the file path is empty, we will overwrite it with the pre-defined value. - target = t - } results = append(results, types.Result{ - Target: target, + Target: targetName(app.Type, app.FilePath), Class: types.ClassLangPkg, Type: app.Type, Packages: app.Libraries, @@ -55,8 +50,8 @@ func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) ty return results } -func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types.Results, error) { - apps := detail.Applications +func (s *scanner) Scan(target types.ScanTarget, _ types.ScanOptions) (types.Results, error) { + apps := target.Applications log.Logger.Infof("Number of language-specific files: %d", len(apps)) if len(apps) == 0 { return nil, nil @@ -83,14 +78,8 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types continue } - target := app.FilePath - if t, ok := PkgTargets[app.Type]; ok && target == "" { - // When the file path is empty, we will overwrite it with the pre-defined value. - target = t - } - results = append(results, types.Result{ - Target: target, + Target: targetName(app.Type, app.FilePath), Vulnerabilities: vulns, Class: types.ClassLangPkg, Type: app.Type, @@ -101,3 +90,11 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types }) return results, nil } + +func targetName(appType ftypes.LangType, filePath string) string { + if t, ok := PkgTargets[appType]; ok && filePath == "" { + // When the file path is empty, we will overwrite it with the pre-defined value. + return t + } + return filePath +} diff --git a/pkg/scanner/local/scan.go b/pkg/scanner/local/scan.go index 5539bf23efc0..63822852cb71 100644 --- a/pkg/scanner/local/scan.go +++ b/pkg/scanner/local/scan.go @@ -57,24 +57,25 @@ func NewScanner(a applier.Applier, osPkgScanner ospkg.Scanner, langPkgScanner la } // Scan scans the artifact and return results. -func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys []string, options types.ScanOptions) (types.Results, ftypes.OS, error) { - artifactDetail, err := s.applier.ApplyLayers(artifactKey, blobKeys) +func (s Scanner) Scan(ctx context.Context, targetName, artifactKey string, blobKeys []string, options types.ScanOptions) ( + types.Results, ftypes.OS, error) { + detail, err := s.applier.ApplyLayers(artifactKey, blobKeys) switch { case errors.Is(err, analyzer.ErrUnknownOS): log.Logger.Debug("OS is not detected.") // Packages may contain OS-independent binary information even though OS is not detected. - if len(artifactDetail.Packages) != 0 { - artifactDetail.OS = ftypes.OS{Family: "none"} + if len(detail.Packages) != 0 { + detail.OS = ftypes.OS{Family: "none"} } // If OS is not detected and repositories are detected, we'll try to use repositories as OS. - if artifactDetail.Repository != nil { - log.Logger.Debugf("Package repository: %s %s", artifactDetail.Repository.Family, artifactDetail.Repository.Release) - log.Logger.Debugf("Assuming OS is %s %s.", artifactDetail.Repository.Family, artifactDetail.Repository.Release) - artifactDetail.OS = ftypes.OS{ - Family: artifactDetail.Repository.Family, - Name: artifactDetail.Repository.Release, + if detail.Repository != nil { + log.Logger.Debugf("Package repository: %s %s", detail.Repository.Family, detail.Repository.Release) + log.Logger.Debugf("Assuming OS is %s %s.", detail.Repository.Family, detail.Repository.Release) + detail.OS = ftypes.OS{ + Family: detail.Repository.Family, + Name: detail.Repository.Release, } } case errors.Is(err, analyzer.ErrNoPkgsDetected): @@ -84,29 +85,46 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys return nil, ftypes.OS{}, xerrors.Errorf("failed to apply layers: %w", err) } + target := types.ScanTarget{ + Name: targetName, + OS: detail.OS, + Repository: detail.Repository, + Packages: mergePkgs(detail.Packages, detail.ImageConfig.Packages, options), + Applications: detail.Applications, + Misconfigurations: mergeMisconfigurations(targetName, detail), + Secrets: mergeSecrets(targetName, detail), + Licenses: detail.Licenses, + CustomResources: detail.CustomResources, + } + + return s.ScanTarget(ctx, target, options) +} + +func (s Scanner) ScanTarget(ctx context.Context, target types.ScanTarget, options types.ScanOptions) (types.Results, ftypes.OS, error) { var eosl bool var results, pkgResults types.Results + var err error // By default, we need to remove dev dependencies from the result // IncludeDevDeps option allows you not to remove them - excludeDevDeps(artifactDetail.Applications, options.IncludeDevDeps) + excludeDevDeps(target.Applications, options.IncludeDevDeps) // Fill OS packages and language-specific packages if options.ListAllPackages { - if res := s.osPkgScanner.Packages(target, artifactDetail, options); len(res.Packages) != 0 { + if res := s.osPkgScanner.Packages(target, options); len(res.Packages) != 0 { pkgResults = append(pkgResults, res) } - pkgResults = append(pkgResults, s.langPkgScanner.Packages(artifactDetail, options)...) + pkgResults = append(pkgResults, s.langPkgScanner.Packages(target, options)...) } // Scan packages for vulnerabilities if options.Scanners.Enabled(types.VulnerabilityScanner) { var vulnResults types.Results - vulnResults, eosl, err = s.scanVulnerabilities(target, artifactDetail, options) + vulnResults, eosl, err = s.scanVulnerabilities(target, options) if err != nil { return nil, ftypes.OS{}, xerrors.Errorf("failed to detect vulnerabilities: %w", err) } - artifactDetail.OS.Eosl = eosl + target.OS.Eosl = eosl // Merge package results into vulnerability results mergedResults := s.fillPkgsInVulns(pkgResults, vulnResults) @@ -117,45 +135,20 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys results = append(results, pkgResults...) } - // Scan IaC config files - if ShouldScanMisconfigOrRbac(options.Scanners) { - configResults := s.MisconfsToResults(artifactDetail.Misconfigurations) - results = append(results, configResults...) - } + // Store misconfigurations + results = append(results, s.misconfsToResults(target.Misconfigurations, options)...) - // Scan secrets - if options.Scanners.Enabled(types.SecretScanner) { - secretResults := s.secretsToResults(artifactDetail.Secrets) - results = append(results, secretResults...) - } + // Store secrets + results = append(results, s.secretsToResults(target.Secrets, options)...) // Scan licenses - if options.Scanners.Enabled(types.LicenseScanner) { - licenseResults := s.scanLicenses(artifactDetail, options.LicenseCategories) - results = append(results, licenseResults...) - } - - // Scan misconfigurations on container image config - if options.ImageConfigScanners.Enabled(types.MisconfigScanner) { - if im := artifactDetail.ImageConfig.Misconfiguration; im != nil { - im.FilePath = target // Set the target name to the file path as container image config is not a real file. - results = append(results, s.MisconfsToResults([]ftypes.Misconfiguration{*im})...) - } - } - - // Scan secrets on container image config - if options.ImageConfigScanners.Enabled(types.SecretScanner) { - if is := artifactDetail.ImageConfig.Secret; is != nil { - is.FilePath = target // Set the target name to the file path as container image config is not a real file. - results = append(results, s.secretsToResults([]ftypes.Secret{*is})...) - } - } + results = append(results, s.scanLicenses(target, options)...) // For WASM plugins and custom analyzers - if len(artifactDetail.CustomResources) != 0 { + if len(target.CustomResources) != 0 { results = append(results, types.Result{ Class: types.ClassCustom, - CustomResources: artifactDetail.CustomResources, + CustomResources: target.CustomResources, }) } @@ -170,16 +163,16 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys return nil, ftypes.OS{}, xerrors.Errorf("post scan error: %w", err) } - return results, artifactDetail.OS, nil + return results, target.OS, nil } -func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) ( +func (s Scanner) scanVulnerabilities(target types.ScanTarget, options types.ScanOptions) ( types.Results, bool, error) { var eosl bool var results types.Results if slices.Contains(options.VulnType, types.VulnTypeOS) { - vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, detail, options) + vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, options) if err != nil { return nil, false, xerrors.Errorf("unable to scan OS packages: %w", err) } else if vuln.Target != "" { @@ -189,7 +182,7 @@ func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail } if slices.Contains(options.VulnType, types.VulnTypeLibrary) { - vulns, err := s.langPkgScanner.Scan(detail, options) + vulns, err := s.langPkgScanner.Scan(target, options) if err != nil { return nil, false, xerrors.Errorf("failed to scan application libraries: %w", err) } @@ -217,6 +210,14 @@ func (s Scanner) fillPkgsInVulns(pkgResults, vulnResults types.Results) types.Re return results } +func (s Scanner) misconfsToResults(misconfs []ftypes.Misconfiguration, options types.ScanOptions) types.Results { + if !ShouldScanMisconfigOrRbac(options.Scanners) { + return nil + } + + return s.MisconfsToResults(misconfs) +} + // MisconfsToResults is exported for trivy-plugin-aqua purposes only func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Results { log.Logger.Infof("Detected config files: %d", len(misconfs)) @@ -254,7 +255,11 @@ func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Res return results } -func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results { +func (s Scanner) secretsToResults(secrets []ftypes.Secret, options types.ScanOptions) types.Results { + if !options.Scanners.Enabled(types.SecretScanner) { + return nil + } + var results types.Results for _, secret := range secrets { log.Logger.Debugf("Secret file: %s", secret.FilePath) @@ -268,15 +273,17 @@ func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results { return results } -func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, - categories map[ftypes.LicenseCategory][]string) types.Results { - scanner := licensing.NewScanner(categories) +func (s Scanner) scanLicenses(target types.ScanTarget, options types.ScanOptions) types.Results { + if !options.Scanners.Enabled(types.LicenseScanner) { + return nil + } var results types.Results + scanner := licensing.NewScanner(options.LicenseCategories) // License - OS packages var osPkgLicenses []types.DetectedLicense - for _, pkg := range detail.Packages { + for _, pkg := range target.Packages { for _, license := range pkg.Licenses { category, severity := scanner.Scan(license) osPkgLicenses = append(osPkgLicenses, types.DetectedLicense{ @@ -296,7 +303,7 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, }) // License - language-specific packages - for _, app := range detail.Applications { + for _, app := range target.Applications { var langLicenses []types.DetectedLicense for _, lib := range app.Libraries { for _, license := range lib.Licenses { @@ -311,13 +318,13 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, } } - target := app.FilePath - if t, ok := langpkg.PkgTargets[app.Type]; ok && target == "" { + targetName := app.FilePath + if t, ok := langpkg.PkgTargets[app.Type]; ok && targetName == "" { // When the file path is empty, we will overwrite it with the pre-defined value. - target = t + targetName = t } results = append(results, types.Result{ - Target: target, + Target: targetName, Class: types.ClassLicense, Licenses: langLicenses, }) @@ -325,7 +332,7 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail, // License - file header or license file var fileLicenses []types.DetectedLicense - for _, license := range detail.Licenses { + for _, license := range target.Licenses { for _, finding := range license.Findings { category, severity := scanner.Scan(finding.Name) fileLicenses = append(fileLicenses, types.DetectedLicense{ @@ -420,3 +427,46 @@ func excludeDevDeps(apps []ftypes.Application, include bool) { }) } } + +func mergePkgs(pkgs, pkgsFromCommands []ftypes.Package, options types.ScanOptions) []ftypes.Package { + if !options.ScanRemovedPackages || len(pkgsFromCommands) == 0 { + return pkgs + } + + // pkg has priority over pkgsFromCommands + uniqPkgs := make(map[string]struct{}) + for _, pkg := range pkgs { + uniqPkgs[pkg.Name] = struct{}{} + } + for _, pkg := range pkgsFromCommands { + if _, ok := uniqPkgs[pkg.Name]; ok { + continue + } + pkgs = append(pkgs, pkg) + } + return pkgs +} + +// mergeMisconfigurations merges misconfigurations on container image config +func mergeMisconfigurations(targetName string, detail ftypes.ArtifactDetail) []ftypes.Misconfiguration { + if detail.ImageConfig.Misconfiguration == nil { + return detail.Misconfigurations + } + + // Append misconfigurations on container image config + misconf := detail.ImageConfig.Misconfiguration + misconf.FilePath = targetName // Set the target name to the file path as container image config is not a real file. + return append(detail.Misconfigurations, *misconf) +} + +// mergeSecrets merges secrets on container image config. +func mergeSecrets(targetName string, detail ftypes.ArtifactDetail) []ftypes.Secret { + if detail.ImageConfig.Secret == nil { + return detail.Secrets + } + + // Append secrets on container image config + secret := detail.ImageConfig.Secret + secret.FilePath = targetName // Set the target name to the file path as container image config is not a real file. + return append(detail.Secrets, *secret) +} diff --git a/pkg/scanner/ospkg/scan.go b/pkg/scanner/ospkg/scan.go index ed255dd362e3..a07d6e6b2091 100644 --- a/pkg/scanner/ospkg/scan.go +++ b/pkg/scanner/ospkg/scan.go @@ -8,14 +8,13 @@ import ( "golang.org/x/xerrors" ospkgDetector "github.com/aquasecurity/trivy/pkg/detector/ospkg" - ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/types" ) type Scanner interface { - Packages(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) types.Result - Scan(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Result, bool, error) + Packages(target types.ScanTarget, options types.ScanOptions) types.Result + Scan(target types.ScanTarget, options types.ScanOptions) (types.Result, bool, error) } type scanner struct{} @@ -24,68 +23,45 @@ func NewScanner() Scanner { return &scanner{} } -func (s *scanner) Packages(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) types.Result { - if len(detail.Packages) == 0 || !detail.OS.Detected() { +func (s *scanner) Packages(target types.ScanTarget, _ types.ScanOptions) types.Result { + if len(target.Packages) == 0 || !target.OS.Detected() { return types.Result{} } - pkgs := detail.Packages - if options.ScanRemovedPackages { - pkgs = mergePkgs(pkgs, detail.ImageConfig.Packages) - } - sort.Sort(pkgs) + sort.Sort(target.Packages) return types.Result{ - Target: fmt.Sprintf("%s (%s %s)", target, detail.OS.Family, detail.OS.Name), + Target: fmt.Sprintf("%s (%s %s)", target.Name, target.OS.Family, target.OS.Name), Class: types.ClassOSPkg, - Type: detail.OS.Family, - Packages: pkgs, + Type: target.OS.Family, + Packages: target.Packages, } } -func (s *scanner) Scan(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Result, bool, error) { - if !detail.OS.Detected() { +func (s *scanner) Scan(target types.ScanTarget, _ types.ScanOptions) (types.Result, bool, error) { + if !target.OS.Detected() { log.Logger.Debug("Detected OS: unknown") return types.Result{}, false, nil } - log.Logger.Infof("Detected OS: %s", detail.OS.Family) - - pkgs := detail.Packages - if options.ScanRemovedPackages { - pkgs = mergePkgs(pkgs, detail.ImageConfig.Packages) - } + log.Logger.Infof("Detected OS: %s", target.OS.Family) - if detail.OS.Extended { + if target.OS.Extended { // TODO: move the logic to each detector - detail.OS.Name += "-ESM" + target.OS.Name += "-ESM" } - vulns, eosl, err := ospkgDetector.Detect("", detail.OS.Family, detail.OS.Name, detail.Repository, time.Time{}, pkgs) + vulns, eosl, err := ospkgDetector.Detect("", target.OS.Family, target.OS.Name, target.Repository, time.Time{}, + target.Packages) if err == ospkgDetector.ErrUnsupportedOS { return types.Result{}, false, nil } else if err != nil { return types.Result{}, false, xerrors.Errorf("failed vulnerability detection of OS packages: %w", err) } - artifactDetail := fmt.Sprintf("%s (%s %s)", target, detail.OS.Family, detail.OS.Name) + artifactDetail := fmt.Sprintf("%s (%s %s)", target.Name, target.OS.Family, target.OS.Name) return types.Result{ Target: artifactDetail, Vulnerabilities: vulns, Class: types.ClassOSPkg, - Type: detail.OS.Family, + Type: target.OS.Family, }, eosl, nil } - -func mergePkgs(pkgs, pkgsFromCommands []ftypes.Package) []ftypes.Package { - // pkg has priority over pkgsFromCommands - uniqPkgs := make(map[string]struct{}) - for _, pkg := range pkgs { - uniqPkgs[pkg.Name] = struct{}{} - } - for _, pkg := range pkgsFromCommands { - if _, ok := uniqPkgs[pkg.Name]; ok { - continue - } - pkgs = append(pkgs, pkg) - } - return pkgs -} diff --git a/pkg/types/scan.go b/pkg/types/scan.go new file mode 100644 index 000000000000..0b5833b12faa --- /dev/null +++ b/pkg/types/scan.go @@ -0,0 +1,33 @@ +package types + +import ( + "github.com/aquasecurity/trivy/pkg/fanal/types" +) + +// ScanTarget holds the attributes for scanning. +type ScanTarget struct { + Name string // container image name, file path, etc + OS types.OS + Repository *types.Repository + Packages types.Packages + Applications []types.Application + Misconfigurations []types.Misconfiguration + Secrets []types.Secret + Licenses []types.LicenseFile + + // CustomResources hold analysis results from custom analyzers. + // It is for extensibility and not used in OSS. + CustomResources []types.CustomResource +} + +// ScanOptions holds the attributes for scanning vulnerabilities +type ScanOptions struct { + VulnType []string + Scanners Scanners + ImageConfigScanners Scanners // Scanners for container image configuration + ScanRemovedPackages bool + ListAllPackages bool + LicenseCategories map[types.LicenseCategory][]string + FilePatterns []string + IncludeDevDeps bool +} diff --git a/pkg/types/scanoptions.go b/pkg/types/scanoptions.go deleted file mode 100644 index 1e075bba6839..000000000000 --- a/pkg/types/scanoptions.go +++ /dev/null @@ -1,17 +0,0 @@ -package types - -import ( - "github.com/aquasecurity/trivy/pkg/fanal/types" -) - -// ScanOptions holds the attributes for scanning vulnerabilities -type ScanOptions struct { - VulnType []string - Scanners Scanners - ImageConfigScanners Scanners // Scanners for container image configuration - ScanRemovedPackages bool - ListAllPackages bool - LicenseCategories map[types.LicenseCategory][]string - FilePatterns []string - IncludeDevDeps bool -} From 1a15a3adb1e42a6e3e3ab8706f3262a4a80892d0 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Fri, 20 Oct 2023 15:16:23 +0600 Subject: [PATCH 06/14] feat(dart): add graph support (#5374) Signed-off-by: knqyf263 Co-authored-by: knqyf263 --- docs/docs/configuration/reporting.md | 65 ++++--- docs/docs/coverage/language/dart.md | 7 +- .../analyzer/language/dart/pub/pubspec.go | 154 +++++++++++++++- .../language/dart/pub/pubspec_test.go | 169 ++++++++++++++---- .../{broken.lock => broken/pubspec.lock} | 0 .../{empty.lock => empty/pubspec.lock} | 0 .../language/dart/pub/testdata/happy.lock | 25 --- .../pub.dev/collection-1.18.0/pubspec.yaml | 16 ++ .../hosted/pub.dev/crypto-3.0.3/pubspec.yaml | 17 ++ .../hosted/pub.dev/meta-1.11.0/pubspec.yaml | 17 ++ .../pub.dev/typed_data-1.3.2/pubspec.yaml | 18 ++ .../dart/pub/testdata/happy/pubspec.lock | 37 ++++ 12 files changed, 428 insertions(+), 97 deletions(-) rename pkg/fanal/analyzer/language/dart/pub/testdata/{broken.lock => broken/pubspec.lock} (100%) rename pkg/fanal/analyzer/language/dart/pub/testdata/{empty.lock => empty/pubspec.lock} (100%) delete mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock diff --git a/docs/docs/configuration/reporting.md b/docs/docs/configuration/reporting.md index 951fd779cd55..17aa6fd973ec 100644 --- a/docs/docs/configuration/reporting.md +++ b/docs/docs/configuration/reporting.md @@ -41,32 +41,31 @@ In some cases, vulnerable dependencies are not linked directly, and it requires To make this task simpler Trivy can show a dependency origin tree with the `--dependency-tree` flag. This flag is only available with the `--format table` flag. -The following packages/languages are currently supported: - -- OS packages - - apk - - dpkg - - rpm -- Node.js - - npm: package-lock.json - - pnpm: pnpm-lock.yaml - - yarn: yarn.lock -- .NET - - NuGet: packages.lock.json -- Python - - Poetry: poetry.lock -- Ruby - - Bundler: Gemfile.lock -- Rust - - Binaries built with [cargo-auditable][cargo-auditable] -- Go - - Modules: go.mod -- PHP - - Composer -- Java - - Maven: pom.xml - -This tree is the reverse of the npm list command. +The following OS package managers are currently supported: + +| OS Package Managers | +|---------------------| +| apk | +| dpkg | +| rpm | + +The following languages are currently supported: + +| Language | File | +|----------|--------------------------------------------| +| Node.js | [package-lock.json][nodejs-package-lock] | +| | [pnpm-lock.yaml][pnpm-lock] | +| | [yarn.lock][yarn-lock] | +| .NET | [packages.lock.json][dotnet-packages-lock] | +| Python | [poetry.lock][poetry-lock] | +| Ruby | [Gemfile.lock][gemfile-lock] | +| Rust | [cargo-auditable binaries][cargo-binaries] | +| Go | [go.mod][go-mod] | +| PHP | [composer.lock][composer-lock] | +| Java | [pom.xml][pom-xml] | +| Dart | [pubspec.lock][pubspec-lock] | + +This tree is the reverse of the dependency graph. However, if you want to resolve a vulnerability in a particular indirect dependency, the reversed tree is useful to know where that dependency comes from and identify which package you actually need to update. In table output, it looks like: @@ -408,4 +407,16 @@ $ trivy convert --format table --severity CRITICAL result.json [github-sbom-submit]: https://docs.github.com/en/rest/dependency-graph/dependency-submission?apiVersion=2022-11-28#create-a-snapshot-of-dependencies-for-a-repository [os_packages]: ../scanner/vulnerability.md#os-packages -[language_packages]: ../scanner/vulnerability.md#language-specific-packages \ No newline at end of file +[language_packages]: ../scanner/vulnerability.md#language-specific-packages + +[nodejs-package-lock]: ../coverage/language/nodejs.md#npm +[pnpm-lock]: ../coverage/language/nodejs.md#pnpm +[yarn-lock]: ../coverage/language/nodejs.md#yarn +[dotnet-packages-lock]: ../coverage/language/dotnet.md#packageslockjson +[poetry-lock]: ../coverage/language/python.md#poetry +[gemfile-lock]: ../coverage/language/ruby.md#bundler +[go-mod]: ../coverage/language/golang.md#go-modules +[composer-lock]: ../coverage/language/php.md#composer +[pom-xml]: ../coverage/language/java.md#pomxml +[pubspec-lock]: ../coverage/language/dart.md#dart +[cargo-binaries]: ../coverage/language/rust.md#binaries \ No newline at end of file diff --git a/docs/docs/coverage/language/dart.md b/docs/docs/coverage/language/dart.md index f7bba0a3951f..0ce6d1cc52b0 100644 --- a/docs/docs/coverage/language/dart.md +++ b/docs/docs/coverage/language/dart.md @@ -13,7 +13,7 @@ The following table provides an outline of the features Trivy offers. | Package manager | File | Transitive dependencies | Dev dependencies | [Dependency graph][dependency-graph] | Position | |-------------------------|--------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:| -| [Dart][dart-repository] | pubspec.lock | ✓ | Included | - | - | +| [Dart][dart-repository] | pubspec.lock | ✓ | Included | ✓ | - | ## Dart In order to detect dependencies, Trivy searches for `pubspec.lock`. @@ -21,6 +21,11 @@ In order to detect dependencies, Trivy searches for `pubspec.lock`. Trivy marks indirect dependencies, but `pubspec.lock` file doesn't have options to separate root and dev transitive dependencies. So Trivy includes all dependencies in report. +To build `dependency tree` Trivy parses [cache directory][cache-directory]. Currently supported default directories and `PUB_CACHE` environment (absolute path only). +!!! note + Make sure the cache directory contains all the dependencies installed in your application. To download missing dependencies, use `dart pub get` command. + [dart]: https://dart.dev/ [dart-repository]: https://pub.dev/ [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies +[cache-directory]: https://dart.dev/tools/pub/glossary#system-cache diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 5961d9ae2a77..221fe73e6b1a 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -2,35 +2,171 @@ package pub import ( "context" + "io" + "io/fs" "os" "path/filepath" + "runtime" + "sort" + "github.com/samber/lo" + "golang.org/x/exp/maps" "golang.org/x/xerrors" + "gopkg.in/yaml.v3" "github.com/aquasecurity/go-dep-parser/pkg/dart/pub" + godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types" + "github.com/aquasecurity/go-dep-parser/pkg/utils" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) func init() { - analyzer.RegisterAnalyzer(&pubSpecLockAnalyzer{}) + analyzer.RegisterPostAnalyzer(analyzer.TypePubSpecLock, newPubSpecLockAnalyzer) } const ( - version = 1 + version = 2 + pubSpecYamlFileName = "pubspec.yaml" ) -// pubSpecLockAnalyzer analyzes pubspec.lock -type pubSpecLockAnalyzer struct{} +// pubSpecLockAnalyzer analyzes `pubspec.lock` +type pubSpecLockAnalyzer struct { + parser godeptypes.Parser +} + +func newPubSpecLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { + return pubSpecLockAnalyzer{ + parser: pub.NewParser(), + }, nil +} -func (a pubSpecLockAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { - p := pub.NewParser() - res, err := language.Analyze(types.Pub, input.FilePath, input.Content, p) +func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { + var apps []types.Application + + // get all DependsOn from cache dir + // lib ID -> DependsOn names + allDependsOn, err := findDependsOn() if err != nil { - return nil, xerrors.Errorf("%s parse error: %w", input.FilePath, err) + log.Logger.Warnf("Unable to parse cache dir: %s", err) + } + + required := func(path string, d fs.DirEntry) bool { + return filepath.Base(path) == types.PubSpecLock + } + + err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error { + app, err := language.Parse(types.Pub, path, r, a.parser) + if err != nil { + return xerrors.Errorf("unable to parse %q: %w", path, err) + } + + if app == nil { + return nil + } + + if allDependsOn != nil { + // Required to search for library versions for DependsOn. + libs := lo.SliceToMap(app.Libraries, func(lib types.Package) (string, string) { + return lib.Name, lib.ID + }) + + for i, lib := range app.Libraries { + var dependsOn []string + for _, depName := range allDependsOn[lib.ID] { + if depID, ok := libs[depName]; ok { + dependsOn = append(dependsOn, depID) + } + } + app.Libraries[i].DependsOn = dependsOn + } + } + + sort.Sort(app.Libraries) + apps = append(apps, *app) + return nil + }) + if err != nil { + return nil, xerrors.Errorf("walk error: %w", err) + } + + return &analyzer.AnalysisResult{ + Applications: apps, + }, nil +} + +func findDependsOn() (map[string][]string, error) { + dir := cacheDir() + if !fsutils.DirExists(dir) { + log.Logger.Debugf("Cache dir (%s) not found. Need 'dart pub get' to fill dependency relationships", dir) + return nil, nil + } + + required := func(path string, d fs.DirEntry) bool { + return filepath.Base(path) == pubSpecYamlFileName } - return res, nil + + deps := make(map[string][]string) + if err := fsutils.WalkDir(os.DirFS(dir), ".", required, func(path string, d fs.DirEntry, r io.Reader) error { + id, dependsOn, err := parsePubSpecYaml(r) + if err != nil { + log.Logger.Debugf("Unable to parse %q: %s", path, err) + return nil + } + if id != "" { + deps[id] = dependsOn + } + return nil + + }); err != nil { + return nil, xerrors.Errorf("walk error: %w", err) + } + return deps, nil +} + +// https://dart.dev/tools/pub/glossary#system-cache +func cacheDir() string { + if dir := os.Getenv("PUB_CACHE"); dir != "" { + return dir + } + + // `%LOCALAPPDATA%\Pub\Cache` for Windows + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("LOCALAPPDATA"), "Pub", "Cache") + } + + // `~/.pub-cache` for Linux or Mac + return filepath.Join(os.Getenv("HOME"), ".pub_cache") +} + +type pubSpecYaml struct { + Name string `yaml:"name"` + Version string `yaml:"version,omitempty"` + Dependencies map[string]interface{} `yaml:"dependencies,omitempty"` +} + +func parsePubSpecYaml(r io.Reader) (string, []string, error) { + var spec pubSpecYaml + if err := yaml.NewDecoder(r).Decode(&spec); err != nil { + return "", nil, xerrors.Errorf("unable to decode: %w", err) + } + + // Version is a required field only for packages from pub.dev: + // https://dart.dev/tools/pub/pubspec#version + // We can skip packages without version, + // because we compare packages by ID (name+version) + if spec.Version == "" || len(spec.Dependencies) == 0 { + return "", nil, nil + } + + // pubspec.yaml uses version ranges + // save only dependencies names + dependsOn := maps.Keys(spec.Dependencies) + + return utils.PackageID(spec.Name, spec.Version), dependsOn, nil } func (a pubSpecLockAnalyzer) Required(filePath string, _ os.FileInfo) bool { diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go index e03d9c1c8a64..d8ecad82fb3f 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go @@ -1,8 +1,9 @@ package pub import ( + "context" "os" - "sort" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -14,35 +15,51 @@ import ( func Test_pubSpecLockAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - inputFile string - want *analyzer.AnalysisResult - wantErr assert.ErrorAssertionFunc + name string + dir string + pubCacheEnv string + want *analyzer.AnalysisResult + wantErr assert.ErrorAssertionFunc }{ { - name: "happy path", - inputFile: "testdata/happy.lock", + // Supports only absolute paths for `rootUri` in package_config.json + // But for this test this field was changed + name: "happy path with cache", + dir: "testdata/happy", + pubCacheEnv: "testdata/happy/cache", want: &analyzer.AnalysisResult{ Applications: []types.Application{ { Type: types.Pub, - FilePath: "testdata/happy.lock", + FilePath: "pubspec.lock", Libraries: types.Packages{ { - ID: "crypto@3.0.2", + ID: "collection@1.17.0", + Name: "collection", + Version: "1.17.0", + Indirect: true, + }, + { + ID: "crypto@3.0.3", Name: "crypto", - Version: "3.0.2", + Version: "3.0.3", + DependsOn: []string{ + "typed_data@1.3.2", + }, }, { - ID: "flutter_test@0.0.0", - Name: "flutter_test", - Version: "0.0.0", + ID: "meta@1.11.0", + Name: "meta", + Version: "1.11.0", }, { - ID: "uuid@3.0.6", - Name: "uuid", - Version: "3.0.6", + ID: "typed_data@1.3.2", + Name: "typed_data", + Version: "1.3.2", Indirect: true, + DependsOn: []string{ + "collection@1.17.0", + }, }, }, }, @@ -51,39 +68,121 @@ func Test_pubSpecLockAnalyzer_Analyze(t *testing.T) { wantErr: assert.NoError, }, { - name: "empty file", - inputFile: "testdata/empty.lock", - wantErr: assert.NoError, + name: "happy path without cache", + dir: "testdata/happy", + pubCacheEnv: "testdata/happy/empty", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.Pub, + FilePath: "pubspec.lock", + Libraries: types.Packages{ + { + ID: "collection@1.17.0", + Name: "collection", + Version: "1.17.0", + Indirect: true, + }, + { + ID: "crypto@3.0.3", + Name: "crypto", + Version: "3.0.3", + }, + { + ID: "meta@1.11.0", + Name: "meta", + Version: "1.11.0", + }, + { + ID: "typed_data@1.3.2", + Name: "typed_data", + Version: "1.3.2", + Indirect: true, + }, + }, + }, + }, + }, + wantErr: assert.NoError, }, { - name: "broken file", - inputFile: "testdata/broken.lock", - wantErr: assert.Error, + name: "empty file", + dir: "testdata/empty", + want: &analyzer.AnalysisResult{}, + wantErr: assert.NoError, + }, + { + name: "broken file", + dir: "testdata/broken", + want: &analyzer.AnalysisResult{}, + wantErr: assert.Error, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f, err := os.Open(tt.inputFile) + t.Setenv("PUB_CACHE", tt.pubCacheEnv) + a, err := newPubSpecLockAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) - defer f.Close() - a := pubSpecLockAnalyzer{} - got, err := a.Analyze(nil, analyzer.AnalysisInput{ - FilePath: tt.inputFile, - Content: f, + got, err := a.PostAnalyze(context.Background(), analyzer.PostAnalysisInput{ + FS: os.DirFS(tt.dir), }) - if got != nil { - for _, app := range got.Applications { - sort.Sort(app.Libraries) + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_pubSpecLockAnalyzer_cacheDir(t *testing.T) { + tests := []struct { + name string + pubCacheEnv string + localAppDataEnv string + windowsTest bool + wantDir string + }{ + { + name: "default cache dir for Linux/MacOS", + wantDir: "/root/.pub_cache", + }, + { + name: "default cache dir Windows", + windowsTest: true, + wantDir: "C:\\Users\\User\\AppData\\Local\\Pub\\Cache", + }, + { + name: "PUB_CACHE is used", + pubCacheEnv: "/root/cache", + wantDir: "/root/cache", + }, + { + name: "PUB_CACHE is used in Windows", + pubCacheEnv: "C:\\Cache", + windowsTest: true, + wantDir: "C:\\Cache", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if runtime.GOOS == "windows" { + if !tt.windowsTest { + t.Skipf("This test is not used for %s", runtime.GOOS) + } + t.Setenv("LOCALAPPDATA", "C:\\Users\\User\\AppData\\Local") + } else { + if tt.windowsTest { + t.Skipf("This test is not used for %s", runtime.GOOS) } + t.Setenv("HOME", "/root") } - if !tt.wantErr(t, err, tt.inputFile) { - return - } - assert.Equal(t, tt.want, got) + t.Setenv("PUB_CACHE", tt.pubCacheEnv) + + dir := cacheDir() + assert.Equal(t, tt.wantDir, dir) }) } } diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/broken.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/broken/pubspec.lock similarity index 100% rename from pkg/fanal/analyzer/language/dart/pub/testdata/broken.lock rename to pkg/fanal/analyzer/language/dart/pub/testdata/broken/pubspec.lock diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/empty.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/empty/pubspec.lock similarity index 100% rename from pkg/fanal/analyzer/language/dart/pub/testdata/empty.lock rename to pkg/fanal/analyzer/language/dart/pub/testdata/empty/pubspec.lock diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock deleted file mode 100644 index 3a37840aa3bb..000000000000 --- a/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by pub -# See https://dart.dev/tools/pub/glossary#lockfile -packages: - crypto: - dependency: "direct main" - description: - name: crypto - url: "https://pub.flutter-io.cn" - source: hosted - version: "3.0.2" - flutter_test: - dependency: "direct dev" - description: flutter - source: sdk - version: "0.0.0" - uuid: - dependency: transitive - description: - name: uuid - url: "https://pub.flutter-io.cn" - source: hosted - version: "3.0.6" -sdks: - dart: ">=2.18.0 <3.0.0" - flutter: ">=3.3.0" \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml new file mode 100644 index 000000000000..2e54e00344a6 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml @@ -0,0 +1,16 @@ +name: collection +version: 1.18.0 +description: >- + Collections and utilities functions and classes related to collections. +repository: https://github.com/dart-lang/collection + +topics: + - data-structures + - collections + +environment: + sdk: ">=2.18.0 <4.0.0" + +dev_dependencies: + lints: ^2.0.1 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml new file mode 100644 index 000000000000..29cc438636a8 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml @@ -0,0 +1,17 @@ +name: crypto +version: 3.0.3 +description: Implementations of SHA, MD5, and HMAC cryptographic functions. +repository: https://github.com/dart-lang/crypto +topics: + - crypto + +environment: + sdk: '>=2.19.0 <3.0.0' + +dependencies: + typed_data: ^1.3.0 + +dev_dependencies: + convert: ^3.0.0 + dart_flutter_team_lints: ^1.0.0 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml new file mode 100644 index 000000000000..d4c6f0f86340 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml @@ -0,0 +1,17 @@ +name: meta +# Note, because version `2.0.0` was mistakenly released, the next major version must be `3.x.y`. +version: 1.11.0 +description: >- + Annotations used to express developer intentions that can't otherwise be + deduced by statically analyzing source code. +repository: https://github.com/dart-lang/sdk/tree/main/pkg/meta + +environment: + sdk: ">=2.12.0 <4.0.0" + +# We use 'any' version constraints here as we get our package versions from +# the dart-lang/sdk repo's DEPS file. Note that this is a special case; the +# best practice for packages is to specify their compatible version ranges. +# See also https://dart.dev/tools/pub/dependencies. +dev_dependencies: + lints: any diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml new file mode 100644 index 000000000000..642bf644115c --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml @@ -0,0 +1,18 @@ +name: typed_data +version: 1.3.2 +description: >- + Utility functions and classes related to the dart:typed_data library. +repository: https://github.com/dart-lang/typed_data + +topics: + - data-structures + +environment: + sdk: '>=2.17.0 <4.0.0' + +dependencies: + collection: ^1.15.0 + +dev_dependencies: + dart_flutter_team_lints: ^0.1.0 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock new file mode 100644 index 000000000000..d3bd9670042c --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock @@ -0,0 +1,37 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + collection: + dependency: transitive + description: + name: collection + sha256: cfc915e6923fe5ce6e153b0723c753045de46de1b4d63771530504004a45fae0 + url: "https://pub.dev" + source: hosted + version: "1.17.0" + crypto: + dependency: "direct main" + description: + name: crypto + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab + url: "https://pub.dev" + source: hosted + version: "3.0.3" + meta: + dependency: "direct dev" + description: + name: meta + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 + url: "https://pub.dev" + source: hosted + version: "1.11.0" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + url: "https://pub.dev" + source: hosted + version: "1.3.2" +sdks: + dart: ">=3.1.0 <4.0.0" \ No newline at end of file From 05b3c86a14ba9e0287f5e9fdb9c44a30ea57f4f5 Mon Sep 17 00:00:00 2001 From: yoshinorin Date: Mon, 23 Oct 2023 15:10:39 +0900 Subject: [PATCH 07/14] docs(misconf): fix broken table for `Use container image` section (#5425) --- docs/getting-started/installation.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 0581f1dc1d72..d142931fd462 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -149,10 +149,11 @@ Example: docker run -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/Library/Caches:/root/.cache/ aquasec/trivy:{{ git.tag[1:] }} image python:3.4-alpine ``` -Registry | Repository | Link | Supportability -Docker Hub | `docker.io/aquasec/trivy` | https://hub.docker.com/r/aquasec/trivy | Official -GitHub Container Registry (GHCR) | `ghcr.io/aquasecurity/trivy` | https://github.com/orgs/aquasecurity/packages/container/package/trivy | Official -AWS Elastic Container Registry (ECR) | `public.ecr.aws/aquasecurity/trivy` | https://gallery.ecr.aws/aquasecurity/trivy | Official +| Registry | Repository | Link | Supportability | +|--------------------------------------|-------------------------------------|-----------------------------------------------------------------------|----------------| +| Docker Hub | `docker.io/aquasec/trivy` | https://hub.docker.com/r/aquasec/trivy | Official | +| GitHub Container Registry (GHCR) | `ghcr.io/aquasecurity/trivy` | https://github.com/orgs/aquasecurity/packages/container/package/trivy | Official | +| AWS Elastic Container Registry (ECR) | `public.ecr.aws/aquasecurity/trivy` | https://gallery.ecr.aws/aquasecurity/trivy | Official | ## Other Tools to use and deploy Trivy From 01c98d1516047f6dd41e1d943f0ead3b27e53c93 Mon Sep 17 00:00:00 2001 From: simar7 <1254783+simar7@users.noreply.github.com> Date: Mon, 23 Oct 2023 01:32:08 -0600 Subject: [PATCH 08/14] feat(misconf): Support `--ignore-policy` in config scans (#5359) Signed-off-by: Simar --- .../configuration/cli/trivy_config.md | 1 + pkg/commands/app.go | 1 - pkg/result/filter.go | 25 ++++--- pkg/result/filter_test.go | 67 ++++++++++++++++++- .../testdata/test-ignore-policy-misconf.rego | 9 +++ 5 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 pkg/result/testdata/test-ignore-policy-misconf.rego diff --git a/docs/docs/references/configuration/cli/trivy_config.md b/docs/docs/references/configuration/cli/trivy_config.md index 76a0bf91cb3b..9cb61d2990c2 100644 --- a/docs/docs/references/configuration/cli/trivy_config.md +++ b/docs/docs/references/configuration/cli/trivy_config.md @@ -24,6 +24,7 @@ trivy config [flags] DIR --helm-set-string strings specify Helm string values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) --helm-values strings specify paths to override the Helm values.yaml files -h, --help help for config + --ignore-policy string specify the Rego file path to evaluate each vulnerability --ignorefile string specify .trivyignore file (default ".trivyignore") --include-non-failures include successes and exceptions, available with '--scanners config' --k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0) diff --git a/pkg/commands/app.go b/pkg/commands/app.go index b9937b6b8756..85cf164275ff 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -622,7 +622,6 @@ func NewServerCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { func NewConfigCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { reportFlagGroup := flag.NewReportFlagGroup() reportFlagGroup.DependencyTree = nil // disable '--dependency-tree' - reportFlagGroup.IgnorePolicy = nil // disable '--ignore-policy' reportFlagGroup.ListAllPkgs = nil // disable '--list-all-pkgs' reportFlagGroup.ExitOnEOL = nil // disable '--exit-on-eol' reportFormat := flag.ReportFormatFlag diff --git a/pkg/result/filter.go b/pkg/result/filter.go index 8e2df0b92db9..9dc738d318aa 100644 --- a/pkg/result/filter.go +++ b/pkg/result/filter.go @@ -65,18 +65,24 @@ func FilterResult(ctx context.Context, result *types.Result, ignoreConf IgnoreCo result.Secrets = filterSecrets(result, severities, ignoreConf.Secrets) result.Licenses = filterLicenses(result.Licenses, severities, opt.IgnoreLicenses, ignoreConf.Licenses) + var ignoredMisconfs int if opt.PolicyFile != "" { var err error - filteredVulns, filteredMisconfs, err = applyPolicy(ctx, filteredVulns, filteredMisconfs, opt.PolicyFile) + var ignored int + filteredVulns, filteredMisconfs, ignored, err = applyPolicy(ctx, filteredVulns, filteredMisconfs, opt.PolicyFile) if err != nil { return xerrors.Errorf("failed to apply the policy: %w", err) } + ignoredMisconfs += ignored } sort.Sort(types.BySeverity(filteredVulns)) result.Vulnerabilities = filteredVulns - result.Misconfigurations = filteredMisconfs result.MisconfSummary = misconfSummary + if result.MisconfSummary != nil { + result.MisconfSummary.Exceptions += ignoredMisconfs + } + result.Misconfigurations = filteredMisconfs return nil } @@ -147,6 +153,7 @@ func filterMisconfigurations(result *types.Result, severities []string, includeN continue } else if ignoreMisconfs.Match(result.Target, misconf.ID) || ignoreMisconfs.Match(result.Target, misconf.AVDID) { // Filter misconfigurations by ignore file + summary.Exceptions++ continue } @@ -215,10 +222,10 @@ func summarize(status types.MisconfStatus, summary *types.MisconfSummary) { } func applyPolicy(ctx context.Context, vulns []types.DetectedVulnerability, misconfs []types.DetectedMisconfiguration, - policyFile string) ([]types.DetectedVulnerability, []types.DetectedMisconfiguration, error) { + policyFile string) ([]types.DetectedVulnerability, []types.DetectedMisconfiguration, int, error) { policy, err := os.ReadFile(policyFile) if err != nil { - return nil, nil, xerrors.Errorf("unable to read the policy file: %w", err) + return nil, nil, 0, xerrors.Errorf("unable to read the policy file: %w", err) } query, err := rego.New( @@ -227,7 +234,7 @@ func applyPolicy(ctx context.Context, vulns []types.DetectedVulnerability, misco rego.Module("trivy.rego", string(policy)), ).PrepareForEval(ctx) if err != nil { - return nil, nil, xerrors.Errorf("unable to prepare for eval: %w", err) + return nil, nil, 0, xerrors.Errorf("unable to prepare for eval: %w", err) } // Vulnerabilities @@ -235,7 +242,7 @@ func applyPolicy(ctx context.Context, vulns []types.DetectedVulnerability, misco for _, vuln := range vulns { ignored, err := evaluate(ctx, query, vuln) if err != nil { - return nil, nil, err + return nil, nil, 0, err } if ignored { continue @@ -244,18 +251,20 @@ func applyPolicy(ctx context.Context, vulns []types.DetectedVulnerability, misco } // Misconfigurations + var ignoredMisconfs int var filteredMisconfs []types.DetectedMisconfiguration for _, misconf := range misconfs { ignored, err := evaluate(ctx, query, misconf) if err != nil { - return nil, nil, err + return nil, nil, 0, err } if ignored { + ignoredMisconfs++ continue } filteredMisconfs = append(filteredMisconfs, misconf) } - return filteredVulns, filteredMisconfs, nil + return filteredVulns, filteredMisconfs, ignoredMisconfs, nil } func evaluate(ctx context.Context, query rego.PreparedEvalQuery, input interface{}) (bool, error) { results, err := query.Eval(ctx, rego.EvalInput(input)) diff --git a/pkg/result/filter_test.go b/pkg/result/filter_test.go index 96698a1874cd..afe32726aa43 100644 --- a/pkg/result/filter_test.go +++ b/pkg/result/filter_test.go @@ -383,6 +383,11 @@ func TestFilter(t *testing.T) { { Target: "Dockerfile", Class: types.ClassConfig, + MisconfSummary: &types.MisconfSummary{ + Successes: 0, + Failures: 0, + Exceptions: 1, + }, }, { Secrets: []ftypes.SecretFinding{ @@ -584,7 +589,7 @@ func TestFilter(t *testing.T) { MisconfSummary: &types.MisconfSummary{ Successes: 0, Failures: 1, - Exceptions: 0, + Exceptions: 2, }, Misconfigurations: []types.DetectedMisconfiguration{ { @@ -685,6 +690,66 @@ func TestFilter(t *testing.T) { }, }, }, + { + name: "ignore file for misconf", + args: args{ + report: types.Report{ + Results: types.Results{ + { + Misconfigurations: []types.DetectedMisconfiguration{ + { + ID: "AVD-TEST-0001", + AVDID: "AVD-TEST-0001", + Title: "test-0001", + Description: "foo", + Severity: dbTypes.SeverityHigh.String(), + Status: types.StatusFailure, + }, + { + ID: "AVD-TEST-0002", + AVDID: "AVD-TEST-0002", + Title: "test-0002", + Description: "bar", + Severity: dbTypes.SeverityHigh.String(), + Status: types.StatusPassed, + }, + { // this misconf is ignored + ID: "AVD-TEST-0003", + AVDID: "AVD-TEST-0003", + Title: "test-0003", + Description: "baz", + Severity: dbTypes.SeverityHigh.String(), + Status: types.StatusFailure, + }, + }, + }, + }, + }, + severities: []dbTypes.Severity{dbTypes.SeverityHigh}, + policyFile: "./testdata/test-ignore-policy-misconf.rego", + }, + want: types.Report{ + Results: types.Results{ + { + MisconfSummary: &types.MisconfSummary{ + Successes: 1, + Failures: 2, + Exceptions: 1, + }, + Misconfigurations: []types.DetectedMisconfiguration{ + { + ID: "AVD-TEST-0001", + AVDID: "AVD-TEST-0001", + Title: "test-0001", + Description: "foo", + Severity: dbTypes.SeverityHigh.String(), + Status: types.StatusFailure, + }, + }, + }, + }, + }, + }, { name: "happy path with duplicates, one with empty fixed version", args: args{ diff --git a/pkg/result/testdata/test-ignore-policy-misconf.rego b/pkg/result/testdata/test-ignore-policy-misconf.rego new file mode 100644 index 000000000000..39838ef27210 --- /dev/null +++ b/pkg/result/testdata/test-ignore-policy-misconf.rego @@ -0,0 +1,9 @@ +package trivy + +import data.lib.trivy + +default ignore=false + +ignore { + input.AVDID != "AVD-TEST-0001" +} From 53c9a7d762c1629d4351daf12d28dd871779d70e Mon Sep 17 00:00:00 2001 From: simar7 <1254783+simar7@users.noreply.github.com> Date: Tue, 24 Oct 2023 07:03:02 -0600 Subject: [PATCH 09/14] docs(misconf): Update `--tf-exclude-downloaded-modules` description (#5419) --- docs/docs/coverage/iac/terraform.md | 2 +- docs/docs/references/configuration/cli/trivy_aws.md | 2 +- docs/docs/references/configuration/cli/trivy_config.md | 2 +- docs/docs/references/configuration/cli/trivy_filesystem.md | 2 +- docs/docs/references/configuration/cli/trivy_image.md | 2 +- docs/docs/references/configuration/cli/trivy_kubernetes.md | 2 +- docs/docs/references/configuration/cli/trivy_repository.md | 2 +- docs/docs/references/configuration/cli/trivy_rootfs.md | 2 +- docs/docs/references/configuration/cli/trivy_vm.md | 2 +- pkg/flag/misconf_flags.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/docs/coverage/iac/terraform.md b/docs/docs/coverage/iac/terraform.md index 7d41ea763603..a8dc17a28fae 100644 --- a/docs/docs/coverage/iac/terraform.md +++ b/docs/docs/coverage/iac/terraform.md @@ -35,7 +35,7 @@ trivy conf --tf-vars dev.terraform.tfvars ./infrastructure/tf ### Exclude Downloaded Terraform Modules By default, downloaded modules are also scanned. -If you don't want to scan modules downloaded into the `.terraform` directory, you can use the `--tf-exclude-downloaded-modules` flag. +If you don't want to scan them, you can use the `--tf-exclude-downloaded-modules` flag. ```bash trivy conf --tf-exclude-downloaded-modules ./configs diff --git a/docs/docs/references/configuration/cli/trivy_aws.md b/docs/docs/references/configuration/cli/trivy_aws.md index 455e2abf298b..e7e5651c91d4 100644 --- a/docs/docs/references/configuration/cli/trivy_aws.md +++ b/docs/docs/references/configuration/cli/trivy_aws.md @@ -95,7 +95,7 @@ trivy aws [flags] --skip-policy-update skip fetching rego policy updates --skip-service strings Skip selected AWS Service(s) specified with this flag. Can specify multiple services using --skip-service A --skip-service B etc. -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --trace enable more verbose trace output for custom queries --update-cache Update the cache for the applicable cloud provider instead of using cached results. diff --git a/docs/docs/references/configuration/cli/trivy_config.md b/docs/docs/references/configuration/cli/trivy_config.md index 9cb61d2990c2..8dfc097e3f29 100644 --- a/docs/docs/references/configuration/cli/trivy_config.md +++ b/docs/docs/references/configuration/cli/trivy_config.md @@ -45,7 +45,7 @@ trivy config [flags] DIR --skip-files strings specify the files or glob patterns to skip --skip-policy-update skip fetching rego policy updates -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --trace enable more verbose trace output for custom queries --username strings username. Comma-separated usernames allowed. diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index 8c681b57c6b2..bfeb6302ce40 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -78,7 +78,7 @@ trivy filesystem [flags] PATH --skip-policy-update skip fetching rego policy updates --slow scan over time with lower CPU and memory utilization -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 1effd26cacf4..dd96a620e634 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -99,7 +99,7 @@ trivy image [flags] IMAGE_NAME --skip-policy-update skip fetching rego policy updates --slow scan over time with lower CPU and memory utilization -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index e140d054f270..3fd04a287b69 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -89,7 +89,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg: --skip-policy-update skip fetching rego policy updates --slow scan over time with lower CPU and memory utilization -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --tolerations strings specify node-collector job tolerations (example: key1=value1:NoExecute,key2=value2:NoSchedule) --trace enable more verbose trace output for custom queries diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 6cf418c5780e..984a40a28af6 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -78,7 +78,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --slow scan over time with lower CPU and memory utilization --tag string pass the tag name to be scanned -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index 69bcdb282d5c..bb007e1210f6 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -79,7 +79,7 @@ trivy rootfs [flags] ROOTDIR --skip-policy-update skip fetching rego policy updates --slow scan over time with lower CPU and memory utilization -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index e294f010cfca..2a2067c2c0df 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -70,7 +70,7 @@ trivy vm [flags] VM_IMAGE --skip-java-db-update skip updating Java index database --slow scan over time with lower CPU and memory utilization -t, --template string output template - --tf-exclude-downloaded-modules remove results for downloaded modules in .terraform folder + --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/pkg/flag/misconf_flags.go b/pkg/flag/misconf_flags.go index bf21b9aab642..74e18113433d 100644 --- a/pkg/flag/misconf_flags.go +++ b/pkg/flag/misconf_flags.go @@ -59,7 +59,7 @@ var ( Name: "tf-exclude-downloaded-modules", ConfigName: "misconfiguration.terraform.exclude-downloaded-modules", Default: false, - Usage: "remove results for downloaded modules in .terraform folder", + Usage: "exclude misconfigurations for downloaded terraform modules", } PolicyBundleRepositoryFlag = Flag{ Name: "policy-bundle-repository", From 57fa701a87baea1d8ffe4d68a311ad410024ce8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:42:55 +0900 Subject: [PATCH 10/14] chore(deps): bump google.golang.org/grpc from 1.57.0 to 1.57.1 (#5447) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5d496371fa04..9bfc015fbee1 100644 --- a/go.mod +++ b/go.mod @@ -369,7 +369,7 @@ require ( google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect - google.golang.org/grpc v1.57.0 // indirect + google.golang.org/grpc v1.57.1 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 1fdfe563f603..4a0f12b89b39 100644 --- a/go.sum +++ b/go.sum @@ -2437,8 +2437,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= +google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 26b4959541b8fb10adb5f454ad930c6e2b68a0a8 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Thu, 26 Oct 2023 07:50:32 +0600 Subject: [PATCH 11/14] fix(java): download java-db once (#5442) --- pkg/fanal/analyzer/language/java/jar/jar.go | 15 +++++---------- pkg/javadb/client.go | 12 ++++++++---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/pkg/fanal/analyzer/language/java/jar/jar.go b/pkg/fanal/analyzer/language/java/jar/jar.go index 8e4ed4820f18..0690fc0d9f20 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar.go +++ b/pkg/fanal/analyzer/language/java/jar/jar.go @@ -15,7 +15,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/javadb" - "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/parallel" ) @@ -34,8 +33,7 @@ var requiredExtensions = []string{ // javaLibraryAnalyzer analyzes jar/war/ear/par files type javaLibraryAnalyzer struct { - client *javadb.DB - slow bool + slow bool } func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { @@ -46,23 +44,20 @@ func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnal func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { // TODO: think about the sonatype API and "--offline" - var err error - log.Logger.Info("JAR files found") - a.client, err = javadb.NewClient() + client, err := javadb.NewClient() if err != nil { return nil, xerrors.Errorf("Unable to initialize the Java DB: %s", err) } - defer func() { _ = a.client.Close() }() - log.Logger.Info("Analyzing JAR files takes a while...") + defer func() { _ = client.Close() }() // Skip analyzing JAR files as the nil client means the Java DB was not downloaded successfully. - if a.client == nil { + if client == nil { return nil, nil } // It will be called on each JAR file onFile := func(path string, info fs.FileInfo, r dio.ReadSeekerAt) (*types.Application, error) { - p := jar.NewParser(a.client, jar.WithSize(info.Size()), jar.WithFilePath(path)) + p := jar.NewParser(client, jar.WithSize(info.Size()), jar.WithFilePath(path)) return language.ParsePackage(types.Jar, path, r, p, input.Options.FileChecksum) } diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index f2a68abedb00..e1b67b81f781 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "sync" "time" "golang.org/x/xerrors" @@ -31,6 +32,7 @@ type Updater struct { skip bool quiet bool registryOption ftypes.RegistryOptions + once sync.Once // we need to update java-db once per run } func (u *Updater) Update() error { @@ -93,10 +95,12 @@ func Update() error { if updater == nil { return xerrors.New("Java DB client not initialized") } - if err := updater.Update(); err != nil { - return xerrors.Errorf("Java DB update error: %w", err) - } - return nil + + var err error + updater.once.Do(func() { + err = updater.Update() + }) + return err } type DB struct { From 013d9019938aef50fabaa0aec75f956ffe784aee Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Sat, 28 Oct 2023 09:32:13 +0600 Subject: [PATCH 12/14] docs: fix error when installing `PyYAML` for gh pages (#5462) --- .github/workflows/mkdocs-dev.yaml | 1 + .github/workflows/mkdocs-latest.yaml | 1 + .github/workflows/test-docs.yaml | 1 + docs/build/Dockerfile | 2 +- docs/build/requirements.txt | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mkdocs-dev.yaml b/.github/workflows/mkdocs-dev.yaml index 9c608ca08eec..ed3c650e4995 100644 --- a/.github/workflows/mkdocs-dev.yaml +++ b/.github/workflows/mkdocs-dev.yaml @@ -21,6 +21,7 @@ jobs: python-version: 3.x - name: Install dependencies run: | + python -m pip install --upgrade pip setuptools wheel pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git pip install -r docs/build/requirements.txt env: diff --git a/.github/workflows/mkdocs-latest.yaml b/.github/workflows/mkdocs-latest.yaml index 9996fe7faa66..fd811e31e87e 100644 --- a/.github/workflows/mkdocs-latest.yaml +++ b/.github/workflows/mkdocs-latest.yaml @@ -23,6 +23,7 @@ jobs: python-version: 3.x - name: Install dependencies run: | + python -m pip install --upgrade pip setuptools wheel pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git pip install -r docs/build/requirements.txt env: diff --git a/.github/workflows/test-docs.yaml b/.github/workflows/test-docs.yaml index 43343a9c3fd0..11f4149da86d 100644 --- a/.github/workflows/test-docs.yaml +++ b/.github/workflows/test-docs.yaml @@ -19,6 +19,7 @@ jobs: python-version: 3.x - name: Install dependencies run: | + python -m pip install --upgrade pip setuptools wheel pip install -r docs/build/requirements.txt - name: Configure the git user run: | diff --git a/docs/build/Dockerfile b/docs/build/Dockerfile index 7ebaa1edc0c0..8a90ffcc8fb2 100644 --- a/docs/build/Dockerfile +++ b/docs/build/Dockerfile @@ -1,4 +1,4 @@ -FROM squidfunk/mkdocs-material:8.3.9 +FROM squidfunk/mkdocs-material:9.4.6 ## If you want to see exactly the same version as is published to GitHub pages ## use a private image for insiders, which requires authentication. diff --git a/docs/build/requirements.txt b/docs/build/requirements.txt index 6ca9dba41a1d..68bdb94f2b3a 100644 --- a/docs/build/requirements.txt +++ b/docs/build/requirements.txt @@ -20,7 +20,7 @@ Pygments==2.12.0 pymdown-extensions==9.5 pyparsing==3.0.8 python-dateutil==2.8.2 -PyYAML==6.0 +PyYAML==6.0.1 pyyaml-env-tag==0.1 six==1.16.0 termcolor==1.1.0 From 048150d433bf1512596d86004c80e5ff1a9d3e0b Mon Sep 17 00:00:00 2001 From: Takahiro Tsuruda Date: Sat, 28 Oct 2023 13:06:11 +0900 Subject: [PATCH 13/14] docs: fix the broken link (#5454) Co-authored-by: knqyf263 --- docs/docs/scanner/misconfiguration/custom/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/scanner/misconfiguration/custom/index.md b/docs/docs/scanner/misconfiguration/custom/index.md index 2316668f2ecb..064aa7364f4a 100644 --- a/docs/docs/scanner/misconfiguration/custom/index.md +++ b/docs/docs/scanner/misconfiguration/custom/index.md @@ -201,4 +201,4 @@ See [here](schema.md) for the detail. [rego]: https://www.openpolicyagent.org/docs/latest/policy-language/ [package]: https://www.openpolicyagent.org/docs/latest/policy-language/#packages -[source-types]: https://github.com/aquasecurity/defsec/blob/418759b4dc97af25f30f32e0bd365be7984003a1/pkg/types/sources.go) +[source-types]: https://github.com/aquasecurity/defsec/blob/418759b4dc97af25f30f32e0bd365be7984003a1/pkg/types/sources.go From ca50b77a35670082530a99f41dfcdbac487f5fd1 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:09:36 +0600 Subject: [PATCH 14/14] fix(sbom): save digests for package/application when scanning SBOM files (#5432) Signed-off-by: knqyf263 Co-authored-by: knqyf263 --- pkg/fanal/applier/docker.go | 8 +++ pkg/fanal/applier/docker_test.go | 82 ++++++++++++++++++++++++++++ pkg/sbom/cyclonedx/unmarshal.go | 2 + pkg/sbom/cyclonedx/unmarshal_test.go | 2 + 4 files changed, 94 insertions(+) diff --git a/pkg/fanal/applier/docker.go b/pkg/fanal/applier/docker.go index 824da3a9ea13..b0f42d957eec 100644 --- a/pkg/fanal/applier/docker.go +++ b/pkg/fanal/applier/docker.go @@ -203,6 +203,10 @@ func ApplyLayers(layers []types.BlobInfo) types.ArtifactDetail { } for i, pkg := range mergedLayer.Packages { + // Skip lookup for SBOM + if !lo.IsEmpty(pkg.Layer) { + continue + } originLayerDigest, originLayerDiffID, buildInfo := lookupOriginLayerForPkg(pkg, layers) mergedLayer.Packages[i].Layer = types.Layer{ Digest: originLayerDigest, @@ -218,6 +222,10 @@ func ApplyLayers(layers []types.BlobInfo) types.ArtifactDetail { for _, app := range mergedLayer.Applications { for i, lib := range app.Libraries { + // Skip lookup for SBOM + if !lo.IsEmpty(lib.Layer) { + continue + } originLayerDigest, originLayerDiffID := lookupOriginLayerForLib(app.FilePath, lib, layers) app.Libraries[i].Layer = types.Layer{ Digest: originLayerDigest, diff --git a/pkg/fanal/applier/docker_test.go b/pkg/fanal/applier/docker_test.go index 0e0ae93c8e01..dbd2c3c4a9fb 100644 --- a/pkg/fanal/applier/docker_test.go +++ b/pkg/fanal/applier/docker_test.go @@ -199,6 +199,88 @@ func TestApplyLayers(t *testing.T) { }, }, }, + { + name: "happy path with digests in libs/packages (as for SBOM)", + inputLayers: []types.BlobInfo{ + { + SchemaVersion: 2, + OS: types.OS{ + Family: "debian", + Name: "11.8", + }, + PackageInfos: []types.PackageInfo{ + { + Packages: types.Packages{ + { + ID: "adduser@3.118+deb11u1", + Name: "adduser", + Version: "3.118+deb11u1", + Arch: "all", + SrcName: "adduser", + SrcVersion: "3.118+deb11u1", + Ref: "pkg:deb/debian/adduser@3.118%2Bdeb11u1?arch=all&distro=debian-11.8", + Layer: types.Layer{ + Digest: "sha256:e67fdae3559346105027c63e7fb032bba57e62b1fe9f2da23e6fdfb56384e00b", + DiffID: "sha256:633f5bf471f7595b236a21e62dc60beef321db45916363a02ad5af02d794d497", + }, + }, + }, + }, + }, + Applications: []types.Application{ + { + Type: types.PythonPkg, + Libraries: types.Packages{ + { + Name: "pip", + Version: "23.0.1", + Layer: types.Layer{ + DiffID: "sha256:1def056a3160854c9395aa76282dd62172ec08c18a5fa03bb7d50a777c15ba99", + }, + FilePath: "usr/local/lib/python3.9/site-packages/pip-23.0.1.dist-info/METADATA", + }, + }, + }, + }, + }, + }, + want: types.ArtifactDetail{ + OS: types.OS{ + Family: "debian", + Name: "11.8", + }, + Packages: types.Packages{ + { + ID: "adduser@3.118+deb11u1", + Name: "adduser", + Version: "3.118+deb11u1", + Arch: "all", + SrcName: "adduser", + SrcVersion: "3.118+deb11u1", + Ref: "pkg:deb/debian/adduser@3.118%2Bdeb11u1?arch=all&distro=debian-11.8", + Layer: types.Layer{ + Digest: "sha256:e67fdae3559346105027c63e7fb032bba57e62b1fe9f2da23e6fdfb56384e00b", + DiffID: "sha256:633f5bf471f7595b236a21e62dc60beef321db45916363a02ad5af02d794d497", + }, + }, + }, + Applications: []types.Application{ + { + Type: types.PythonPkg, + Libraries: types.Packages{ + { + Name: "pip", + Version: "23.0.1", + FilePath: "usr/local/lib/python3.9/site-packages/pip-23.0.1.dist-info/METADATA", + Layer: types.Layer{ + DiffID: "sha256:1def056a3160854c9395aa76282dd62172ec08c18a5fa03bb7d50a777c15ba99", + }, + }, + }, + }, + }, + }, + }, { name: "happy path with merging ubuntu version and ESM", inputLayers: []types.BlobInfo{ diff --git a/pkg/sbom/cyclonedx/unmarshal.go b/pkg/sbom/cyclonedx/unmarshal.go index 8282edae2c3f..97c24cf2797a 100644 --- a/pkg/sbom/cyclonedx/unmarshal.go +++ b/pkg/sbom/cyclonedx/unmarshal.go @@ -376,6 +376,8 @@ func toPackage(component cdx.Component) (*purl.PackageURL, *ftypes.Package, erro pkg.Modularitylabel = value case PropertyLayerDiffID: pkg.Layer.DiffID = value + case PropertyLayerDigest: + pkg.Layer.Digest = value case PropertyFilePath: pkg.FilePath = value } diff --git a/pkg/sbom/cyclonedx/unmarshal_test.go b/pkg/sbom/cyclonedx/unmarshal_test.go index 4ea9b17f44c1..26c5e5d7ff49 100644 --- a/pkg/sbom/cyclonedx/unmarshal_test.go +++ b/pkg/sbom/cyclonedx/unmarshal_test.go @@ -221,6 +221,7 @@ func TestUnmarshaler_Unmarshal(t *testing.T) { }, Ref: "pkg:deb/ubuntu/libc6@2.35-0ubuntu3.1?distro=ubuntu-22.04", Layer: ftypes.Layer{ + Digest: "sha256:74ac377868f863e123f24c409f79709f7563fa464557c36a09cf6f85c8b92b7f", DiffID: "sha256:b93c1bd012ab8fda60f5b4f5906bf244586e0e3292d84571d3abb56472248466", }, }, @@ -235,6 +236,7 @@ func TestUnmarshaler_Unmarshal(t *testing.T) { SrcEpoch: 1, Ref: "pkg:deb/ubuntu/libcrypt1@4.4.27-1?epoch=1&distro=ubuntu-22.04", Layer: ftypes.Layer{ + Digest: "sha256:74ac377868f863e123f24c409f79709f7563fa464557c36a09cf6f85c8b92b7f", DiffID: "sha256:b93c1bd012ab8fda60f5b4f5906bf244586e0e3292d84571d3abb56472248466", }, },