From c5c8d7ee262f3fcc1cb91186b02a727991174eee Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Fri, 29 Sep 2023 02:03:29 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20discover=20terraform=20in=20gitlab?= =?UTF-8?q?=20+=20overhaul=20discovery=20(#1975)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Massive overhaul to the discovery code to make it clearer, reduce duplicate calls to the API, and make it ready for groups discovery. It also now discovers Terraform assets, the next PR will have the terraform+git implementation to use these. Signed-off-by: Dominik Richter --- go.sum | 27 --- providers/gitlab/connection/connection.go | 28 +-- providers/gitlab/go.mod | 1 + providers/gitlab/go.sum | 2 + providers/gitlab/provider/discovery.go | 219 ++++++++++++++++++++++ providers/gitlab/provider/provider.go | 178 ++---------------- providers/gitlab/resources/gitlab.lr | 2 +- providers/terraform/provider/provider.go | 6 +- 8 files changed, 258 insertions(+), 205 deletions(-) create mode 100644 providers/gitlab/provider/discovery.go diff --git a/go.sum b/go.sum index 8771775f0d..d2cd064208 100644 --- a/go.sum +++ b/go.sum @@ -73,7 +73,6 @@ github.com/Antonboom/errname v0.1.12/go.mod h1:bK7todrzvlaZoQagP1orKzWXv59X/x0W0 github.com/Antonboom/nilnil v0.1.7 h1:ofgL+BA7vlA1K2wNQOsHzLJ2Pw5B5DpWRLdDAVvvTow= github.com/Antonboom/nilnil v0.1.7/go.mod h1:TP+ScQWVEq0eSIxqU8CbdT5DFWoHp0MbP+KMUO1BKYQ= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -98,13 +97,11 @@ github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0= -github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle v0.3.0 h1:e8vhrYR1nDjzDxyDwpLO27TWOYWilaT+glkwbPadj50= github.com/alecthomas/participle v0.3.0/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= github.com/alecthomas/participle/v2 v2.1.0 h1:z7dElHRrOEEq45F2TG5cbQihMtNTv8vwldytDj7Wrz4= github.com/alecthomas/participle/v2 v2.1.0/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= -github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -117,10 +114,8 @@ github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cv github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY= github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU= github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s= @@ -187,7 +182,6 @@ github.com/breml/bidichk v0.2.6/go.mod h1:Uj07vpi03vZf+TJRNaBzUIcAe/dQ+POeJiuikz github.com/breml/errchkjson v0.3.5 h1:KoiihlE4u+DR1Iq5KloYMFAlQ0UAryuSzc0bkvUgCeQ= github.com/breml/errchkjson v0.3.5/go.mod h1:0ip+8RwJ/ojc1Qkn0YIBqM7A8wnh6GRs/VJx9t9CohE= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= -github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/butuzov/ireturn v0.2.0 h1:kCHi+YzC150GE98WFuZQu9yrTn6GEydO2AuPLbTgnO4= github.com/butuzov/ireturn v0.2.0/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI= @@ -274,7 +268,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= -github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -296,7 +289,6 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y= github.com/firefart/nonamedreturns v1.0.4/go.mod h1:TDhe/tjI1BXo48CmYbUduTV7BdIga8MAO/xbKdcVsGI= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= @@ -306,7 +298,6 @@ github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-critic/go-critic v0.9.0 h1:Pmys9qvU3pSML/3GEQ2Xd9RZ/ip+aXHKILuxczKGV/U= github.com/go-critic/go-critic v0.9.0/go.mod h1:5P8tdXL7m/6qnyG6oRAlYLORvoXH0WDypYgAEmagT40= github.com/go-errors/errors v1.5.0 h1:/EuijeGOu7ckFxzhkj4CXJ8JaenxK7bKUxpPYqeLHqQ= @@ -316,7 +307,6 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -341,7 +331,6 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= @@ -356,7 +345,6 @@ github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlN github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= -github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= @@ -458,7 +446,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -470,7 +457,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= @@ -485,7 +471,6 @@ github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= -github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601 h1:mrEEilTAUmaAORhssPPkxj84TsHrPMLBGW2Z4SoTxm8= github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= @@ -499,7 +484,6 @@ github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3 github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= -github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -556,7 +540,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jgautheron/goconst v1.6.0 h1:gbMLWKRMkzAc6kYsQL6/TxaoBUg3Jm9LSF/Ih1ADWGA= github.com/jgautheron/goconst v1.6.0/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= -github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48= @@ -574,7 +557,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/julz/importas v0.1.0 h1:F78HnrsjY3cR7j0etXy5+TU1Zuy7Xt08X/1aJnH5xXY= @@ -615,7 +597,6 @@ github.com/kulti/thelper v0.6.3/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dq github.com/kunwardeep/paralleltest v1.0.8 h1:Ul2KsqtzFxTlSU7IP0JusWlLiNqQaloB9vguyjbE558= github.com/kunwardeep/paralleltest v1.0.8/go.mod h1:2C7s65hONVqY7Q5Efj5aLzRCNLjw2h4eMc9EcypGjcY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ= github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA= github.com/ldez/gomoddirectives v0.2.3 h1:y7MBaisZVDYmKvt9/l1mjNCiSA1BVn34U0ObUcJwlhA= @@ -697,7 +678,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -724,9 +704,7 @@ github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DV github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= @@ -742,7 +720,6 @@ github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdU github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -855,7 +832,6 @@ github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl01 github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= -github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00= @@ -950,7 +926,6 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t gitlab.com/bosi/decorder v0.4.1 h1:VdsdfxhstabyhZovHafFw+9eJ6eU0d2CkFNJcZz/NU4= gitlab.com/bosi/decorder v0.4.1/go.mod h1:jecSqWUew6Yle1pCr2eLWTensJMmsxHsBwt+PVbkAqA= go-simpler.org/assert v0.6.0 h1:QxSrXa4oRuo/1eHMXSBFHKvJIpWABayzKldqZyugG7E= -go-simpler.org/assert v0.6.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= go.mondoo.com/ranger-rpc v0.5.1 h1:OaVQntDJWDRgbc9nXBP4wOCR9h9Bij8LgHHs86IguGk= go.mondoo.com/ranger-rpc v0.5.1/go.mod h1:3YKcqFrlPgaB4FZ4EoLgdmRtwMQdO7RoAkZYFn+F1eY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -970,7 +945,6 @@ go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5 go.tmz.dev/musttag v0.7.2 h1:1J6S9ipDbalBSODNT5jCep8dhZyMr4ttnjQagmGYR5s= go.tmz.dev/musttag v0.7.2/go.mod h1:m6q5NiiSKMnQYokefa2xGoyoXnrswCbJ0AWYzf4Zs28= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -1406,7 +1380,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/providers/gitlab/connection/connection.go b/providers/gitlab/connection/connection.go index 17fb2f7e3f..1eda257c51 100644 --- a/providers/gitlab/connection/connection.go +++ b/providers/gitlab/connection/connection.go @@ -21,8 +21,8 @@ type GitLabConnection struct { group *gitlab.Group project *gitlab.Project projectID string // only used for initial setup, use project.ID afterwards! - groupPath string - projectPath string + groupName string + projectName string client *gitlab.Client } @@ -60,8 +60,8 @@ func NewGitLabConnection(id uint32, asset *inventory.Asset, conf *inventory.Conf Conf: conf, id: id, asset: asset, - groupPath: conf.Options["group"], - projectPath: conf.Options["project"], + groupName: conf.Options["group"], + projectName: conf.Options["project"], projectID: conf.Options["project-id"], client: client, } @@ -89,28 +89,28 @@ func (c *GitLabConnection) Group() (*gitlab.Group, error) { if c.group != nil { return c.group, nil } - if c.groupPath == "" { - return nil, errors.New("cannot look up gitlab group, no group path defined") + if c.groupName == "" { + return nil, errors.New("cannot look up gitlab group, no group name defined") } var err error - c.group, _, err = c.Client().Groups.GetGroup(c.groupPath, nil) + c.group, _, err = c.Client().Groups.GetGroup(c.groupName, nil) return c.group, err } func (c *GitLabConnection) IsGroup() bool { - return c.groupPath != "" + return c.groupName != "" } func (c *GitLabConnection) IsProject() bool { - return c.projectPath != "" || c.projectID != "" + return c.projectName != "" || c.projectID != "" } func (c *GitLabConnection) GID() (interface{}, error) { - if c.groupPath == "" { + if c.groupName == "" { return nil, errors.New("cannot look up gitlab group, no group path defined") } - return url.QueryEscape(c.groupPath), nil + return url.QueryEscape(c.groupName), nil } func (c *GitLabConnection) PID() (interface{}, error) { @@ -118,13 +118,13 @@ func (c *GitLabConnection) PID() (interface{}, error) { return c.projectID, nil } - if c.groupPath == "" { + if c.groupName == "" { return nil, errors.New("cannot look up gitlab group, no group path defined") } - if c.projectPath == "" { + if c.projectName == "" { return nil, errors.New("cannot look up gitlab project, no project path defined") } - return url.QueryEscape(c.groupPath) + "/" + url.QueryEscape(c.projectPath), nil + return url.QueryEscape(c.groupName) + "/" + url.QueryEscape(c.projectName), nil } func (c *GitLabConnection) Project() (*gitlab.Project, error) { diff --git a/providers/gitlab/go.mod b/providers/gitlab/go.mod index d1123c5348..c5a9d9c1c2 100644 --- a/providers/gitlab/go.mod +++ b/providers/gitlab/go.mod @@ -9,6 +9,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/xanzy/go-gitlab v0.91.1 go.mondoo.com/cnquery v0.0.0-20230920205842-55a158611de3 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( diff --git a/providers/gitlab/go.sum b/providers/gitlab/go.sum index d1a9ba37a9..7ae1b73d88 100644 --- a/providers/gitlab/go.sum +++ b/providers/gitlab/go.sum @@ -287,6 +287,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/providers/gitlab/provider/discovery.go b/providers/gitlab/provider/discovery.go new file mode 100644 index 0000000000..cec3841199 --- /dev/null +++ b/providers/gitlab/provider/discovery.go @@ -0,0 +1,219 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package provider + +import ( + "strconv" + "strings" + + "github.com/rs/zerolog/log" + "github.com/xanzy/go-gitlab" + "go.mondoo.com/cnquery/providers-sdk/v1/inventory" + "go.mondoo.com/cnquery/providers/gitlab/connection" + "golang.org/x/exp/slices" +) + +func (s *Service) discover(root *inventory.Asset, conn *connection.GitLabConnection) (*inventory.Inventory, error) { + if conn.Conf.Discover == nil { + return nil, nil + } + + client := conn.Client() + if client == nil { + return nil, nil + } + + assets := []*inventory.Asset{} + projects := []*gitlab.Project{} + + targets := conn.Conf.Discover.Targets + + // The following calls to discover Groups and Projects will always return + // gitlab.Group and gitlab.Project objects, no matter if we connect to only + // one system or many. This reduces code complexity. + + groupAssets, groups, err := s.discoverGroups(root, conn) + if err != nil { + return nil, err + } + if slices.Contains(targets, DiscoveryGroup) { + assets = append(assets, groupAssets...) + } + + projectAssets, projects, err := s.discoverProjects(root, conn, groups) + if err != nil { + return nil, err + } + if slices.Contains(targets, DiscoveryProject) { + assets = append(assets, projectAssets...) + } + + if slices.Contains(targets, DiscoveryTerraform) { + repos, err := s.discoverTerraform(root, conn, projects) + if err != nil { + return nil, err + } + assets = append(assets, repos...) + } + + if len(assets) == 0 { + return nil, nil + } + return &inventory.Inventory{ + Spec: &inventory.InventorySpec{ + Assets: assets, + }, + }, nil +} + +func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabConnection) ([]*inventory.Asset, []*gitlab.Group, error) { + // If the root asset it a group, we are done because it's the returned + // main asset. If the root is a project, we want to additionally detect + // the group and return it. + // TODO: discover groups for generic gitlab connection + if conn.IsGroup() { + group, err := conn.Group() + return []*inventory.Asset{root}, []*gitlab.Group{group}, err + } + + group, err := conn.Group() + if err != nil { + return nil, nil, err + } + + conf := conn.Conf.Clone() + conf.Type = GitlabGroupConnection + conf.Options = map[string]string{ + "group": group.Name, + "group-id": strconv.Itoa(group.ID), + } + asset := &inventory.Asset{ + Connections: []*inventory.Config{conf}, + } + + s.detectAsGroup(asset, group) + + return []*inventory.Asset{asset}, []*gitlab.Group{group}, nil +} + +func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLabConnection, groups []*gitlab.Group) ([]*inventory.Asset, []*gitlab.Project, error) { + if conn.IsProject() { + project, err := conn.Project() + return []*inventory.Asset{root}, []*gitlab.Project{project}, err + } + + var assets []*inventory.Asset + var projects []*gitlab.Project + + for i := range groups { + group := groups[i] + groupProjects, err := discoverGroupProjects(conn, group.ID) + if err != nil { + return nil, nil, err + } + + for j := range groupProjects { + project := groupProjects[j] + conf := conn.Conf.Clone() + conf.Type = GitlabProjectConnection + conf.Options = map[string]string{ + "group": group.Name, + "group-id": strconv.Itoa(group.ID), + "project": project.Name, + "project-id": strconv.Itoa(project.ID), + } + asset := &inventory.Asset{ + Name: project.NameWithNamespace, + Connections: []*inventory.Config{conf}, + } + + s.detectAsProject(asset, group, project) + if err != nil { + return nil, nil, err + } + + assets = append(assets, asset) + projects = append(projects, project) + } + } + return assets, projects, nil +} + +func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Project, error) { + perPage := 50 + page := 1 + total := 50 + projects := []*gitlab.Project{} + for page*perPage <= total { + projs, resp, err := conn.Client().Groups.ListGroupProjects(gid, &gitlab.ListGroupProjectsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) + if err != nil { + return nil, err + } + projects = append(projects, projs...) + total = resp.TotalItems + page += 1 + } + + return projects, nil +} + +func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) { + var res []*inventory.Asset + for i := range projects { + project := projects[i] + files, err := discoverTerraformHcl(conn.Client(), project.ID) + if err != nil { + log.Error().Err(err).Msg("failed to discover terraform repo in gitlab") + } else if len(files) != 0 { + res = append(res, &inventory.Asset{ + Connections: []*inventory.Config{{ + Type: "terraform-hcl-git", + Options: map[string]string{ + "ssh-url": project.SSHURLToRepo, + "http-url": project.HTTPURLToRepo, + }, + Credentials: conn.Conf.Credentials, + }}, + }) + } + } + return res, nil +} + +// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset +func discoverTerraformHcl(client *gitlab.Client, pid interface{}) ([]string, error) { + opts := &gitlab.ListTreeOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 100, + }, + Recursive: gitlab.Bool(true), + } + + nodes := []*gitlab.TreeNode{} + for { + data, resp, err := client.Repositories.ListTree(pid, opts) + if err != nil { + return nil, err + } + nodes = append(nodes, data...) + + // Exit the loop when we've seen all pages. + if resp.NextPage == 0 { + break + } + + // Update the page number to get the next page. + opts.Page = resp.NextPage + } + + terraformFiles := []string{} + for i := range nodes { + node := nodes[i] + if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") { + terraformFiles = append(terraformFiles, node.Path) + } + } + + return terraformFiles, nil +} diff --git a/providers/gitlab/provider/provider.go b/providers/gitlab/provider/provider.go index fb256a68f7..788d1643e6 100644 --- a/providers/gitlab/provider/provider.go +++ b/providers/gitlab/provider/provider.go @@ -4,7 +4,6 @@ package provider import ( - "context" "errors" "os" "strconv" @@ -199,186 +198,45 @@ var ( } ) -func newGitLabGroupID(groupID string) string { - return "//platformid.api.mondoo.app/runtime/gitlab/group/" + groupID +func newGitLabGroupID(groupID int) string { + return "//platformid.api.mondoo.app/runtime/gitlab/group/" + strconv.Itoa(groupID) } -func newGitLabProjectID(groupID string, projectID string) string { - return "//platformid.api.mondoo.app/runtime/gitlab/group/" + groupID + "/project/" + projectID +func newGitLabProjectID(groupID int, projectID int) string { + return "//platformid.api.mondoo.app/runtime/gitlab/group/" + strconv.Itoa(groupID) + "/project/" + strconv.Itoa(projectID) } func (s *Service) detect(asset *inventory.Asset, conn *connection.GitLabConnection) error { asset.Id = conn.Conf.Type - if conn.IsProject() { - return s.detectAsProject(asset, conn, nil) - } else { - return s.detectAsGroup(asset, conn) - } -} - -func (s *Service) detectAsProject(asset *inventory.Asset, conn *connection.GitLabConnection, project *gitlab.Project) error { - asset.Platform = projectPlatform group, err := conn.Group() if err != nil { return err } - if project == nil { - project, err = conn.Project() + + if conn.IsProject() { + project, err := conn.Project() if err != nil { return err } - } + s.detectAsProject(asset, group, project) - asset.Name = "GitLab Project " + project.Name - asset.PlatformIds = []string{newGitLabProjectID(strconv.Itoa(group.ID), strconv.Itoa(project.ID))} - return nil -} - -func (s *Service) detectAsGroup(asset *inventory.Asset, conn *connection.GitLabConnection) error { - asset.Platform = groupPlatform - group, err := conn.Group() - if err != nil { - return err + } else { + s.detectAsGroup(asset, group) } - asset.Name = "GitLab Group " + group.Name - asset.PlatformIds = []string{newGitLabGroupID(strconv.Itoa(group.ID))} return nil } -func (s *Service) discover(root *inventory.Asset, conn *connection.GitLabConnection) (*inventory.Inventory, error) { - if conn.Conf.Discover == nil { - return nil, nil - } - client := conn.Client() - if client == nil { - return nil, nil - } - - list := []*inventory.Asset{} - targets := conn.Conf.Discover.Targets - - for i := range targets { - target := conn.Conf.Discover.Targets[i] - switch target { - case DiscoveryGroup: - // If the root asset it a group, we are done because it's the returned - // main asset. If the root is a project, we want to additionally detect - // the group and return it. - // TODO: discover groups for generic gitlab connection - if conn.IsGroup() { - list = append(list, root) - continue - } - - conf := conn.Conf.Clone() - conf.Type = GitlabGroupConnection - asset := &inventory.Asset{ - Connections: []*inventory.Config{conf}, - } - err := s.detectAsGroup(asset, conn) - if err != nil { - return nil, err - } - - list = append(list, asset) - - case DiscoveryProject: - // We only discover projects if the root connection is not a project already - if conn.IsProject() { - list = append(list, root) - continue - } - - projects, err := groupProjects(conn) - if err != nil { - return nil, err - } - for _, project := range projects { - conf := conn.Conf.Clone() - conf.Type = GitlabProjectConnection - asset := &inventory.Asset{ - Name: project.NameWithNamespace, - Connections: []*inventory.Config{conf}, - } - err := s.detectAsProject(asset, conn, project) - if err != nil { - return nil, err - } - - list = append(list, asset) - } - } - } - - if len(list) == 0 { - return nil, nil - } - return &inventory.Inventory{ - Spec: &inventory.InventorySpec{ - Assets: list, - }, - }, nil -} - -func groupProjects(conn *connection.GitLabConnection) ([]*gitlab.Project, error) { - gid, err := conn.GID() - if err != nil { - return nil, err - } - - perPage := 50 - page := 1 - total := 50 - projects := []*gitlab.Project{} - for page*perPage <= total { - projs, resp, err := conn.Client().Groups.ListGroupProjects(gid, &gitlab.ListGroupProjectsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) - if err != nil { - return nil, err - } - projects = append(projects, projs...) - total = resp.TotalItems - page += 1 - } - - return projects, nil +func (s *Service) detectAsProject(asset *inventory.Asset, group *gitlab.Group, project *gitlab.Project) { + asset.Platform = projectPlatform + asset.Name = "GitLab Project " + project.Name + asset.PlatformIds = []string{newGitLabProjectID(group.ID, project.ID)} } -// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset -func discoverTerraformHcl(ctx context.Context, client *gitlab.Client, projectId int) ([]string, error) { - opts := &gitlab.ListTreeOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 100, - }, - Recursive: gitlab.Bool(true), - } - - nodes := []*gitlab.TreeNode{} - for { - data, resp, err := client.Repositories.ListTree(projectId, opts) - if err != nil { - return nil, err - } - nodes = append(nodes, data...) - - // Exit the loop when we've seen all pages. - if resp.NextPage == 0 { - break - } - - // Update the page number to get the next page. - opts.Page = resp.NextPage - } - - terraformFiles := []string{} - for i := range nodes { - node := nodes[i] - if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") { - terraformFiles = append(terraformFiles, node.Path) - } - } - - return terraformFiles, nil +func (s *Service) detectAsGroup(asset *inventory.Asset, group *gitlab.Group) { + asset.Platform = groupPlatform + asset.Name = "GitLab Group " + group.Name + asset.PlatformIds = []string{newGitLabGroupID(group.ID)} } func (s *Service) GetData(req *plugin.DataReq) (*plugin.DataRes, error) { diff --git a/providers/gitlab/resources/gitlab.lr b/providers/gitlab/resources/gitlab.lr index 68a027b0b8..63f14a5950 100644 --- a/providers/gitlab/resources/gitlab.lr +++ b/providers/gitlab/resources/gitlab.lr @@ -5,7 +5,7 @@ option provider = "go.mondoo.com/cnquery/providers/gitlab" option go_package = "go.mondoo.com/cnquery/providers/gitlab/resources" // GitLab Group -gitlab.group @defaults("name") { +gitlab.group @defaults("name visibility webURL") { // Group ID id int // Group name diff --git a/providers/terraform/provider/provider.go b/providers/terraform/provider/provider.go index c10334cfa3..651c3985ac 100644 --- a/providers/terraform/provider/provider.go +++ b/providers/terraform/provider/provider.go @@ -17,9 +17,9 @@ import ( ) const ( - StateConnectionType = "state" - PlanConnectionType = "plan" - HclConnectionType = "hcl" + StateConnectionType = "terraform-state" + PlanConnectionType = "terraform-plan" + HclConnectionType = "terraform-hcl" ) type Service struct {