From 87270a7085108907647a3fdd4b85e09a2e0771c2 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 22 Jul 2022 18:11:17 +0800 Subject: [PATCH] Achieve capacity limits --- .gitignore | 1 + api/common/config.go | 10 ++- go.mod | 7 +- go.sum | 163 ++++++++++++++++++++++++++++++++++++ internal/backend.go | 26 +++++- internal/backend_adlv1.go | 8 +- internal/backend_adlv2.go | 4 + internal/backend_azblob.go | 10 ++- internal/backend_gcs3.go | 16 ++-- internal/backend_minio.go | 68 +++++++++++++++ internal/backend_s3.go | 26 +++--- internal/flags.go | 165 +++++++++++++++++++++---------------- internal/goofys.go | 119 +++++++++++++++++--------- 13 files changed, 478 insertions(+), 145 deletions(-) create mode 100644 internal/backend_minio.go diff --git a/.gitignore b/.gitignore index 2ccd54ac..c0be7c5b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ goofys goofys.test xout s3proxy.jar +.vscode \ No newline at end of file diff --git a/api/common/config.go b/api/common/config.go index 092abd87..d4b02a20 100644 --- a/api/common/config.go +++ b/api/common/config.go @@ -27,7 +27,7 @@ import ( ) type PartSizeConfig struct { - PartSize uint64 + PartSize uint64 PartCount uint64 } @@ -45,8 +45,12 @@ type FlagStorage struct { // Common Backend Config UseContentType bool - Endpoint string - Backend interface{} + + Provider string + Capacity uint64 + DiskUsageInterval uint64 + Endpoint string + Backend interface{} // Tuning MemoryLimit uint64 diff --git a/go.mod b/go.mod index 3d0b3cf6..aea93a5e 100644 --- a/go.mod +++ b/go.mod @@ -16,24 +16,21 @@ require ( github.com/aws/aws-sdk-go v1.38.7 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.2 - github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe // indirect github.com/jacobsa/fuse v0.0.0-20210818065549-10d864429bf7 - github.com/jtolds/gls v4.2.0+incompatible // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 // indirect github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect + github.com/minio/madmin-go v1.4.6 github.com/mitchellh/go-homedir v1.1.0 github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b github.com/sevlyar/go-daemon v0.1.5 github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 github.com/sirupsen/logrus v1.8.1 - github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 // indirect - github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c // indirect github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a google.golang.org/api v0.49.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 - gopkg.in/ini.v1 v1.46.0 + gopkg.in/ini.v1 v1.62.0 ) replace github.com/aws/aws-sdk-go => ./s3ext diff --git a/go.sum b/go.sum index 23006bb4..d5f35524 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -84,8 +85,16 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOC github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.38.7 h1:uOu2IrTiNhcSNAjBmA21t48lTx5mgGdcFKamDjXMscA= github.com/aws/aws-sdk-go v1.38.7/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= @@ -100,6 +109,9 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -109,6 +121,7 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -118,8 +131,11 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:Q github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -127,6 +143,10 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -178,6 +198,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -199,19 +220,42 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22 h1:ub2sxhs2A0HRa2dWHa github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe h1:rcf1P0fm+1l0EjG16p06mYLj9gW9X36KgdHJ/88hS4g= github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd h1:9GCSedGjMcLZCrusBZuo4tyKLpKUPenUUqi34AkuFmA= github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff h1:2xRHTvkpJ5zJmglXLRqHiZQNjUoOkhUyhTAhEQvPAWw= @@ -228,15 +272,27 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.0+incompatible h1:f7Me2eb3Wy6xfgSj1uyi0Ho6c4NY1L4KoX7c3auAhOI= github.com/jtolds/gls v4.2.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750 h1:lqGuhK6ejK9x8b6+GPGXm0gzrajIKB7yNKL2fx1oqSU= github.com/kr/pretty v0.1.1-0.20190720101428-71e7e4993750/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -246,38 +302,97 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw= github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/madmin-go v1.4.6 h1:HicA76qv5ilMAjq2aQO6ra5awo8NpcprSH09IEMAjYY= +github.com/minio/madmin-go v1.4.6/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4= +github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= +github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/gopsutil/v3 v3.21.6 h1:vU7jrp1Ic/2sHB7w6UNs7MIkn7ebVtTb5D9j45o9VYE= +github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8 h1:cPg+LIQqTg4YaxZsUtAus45xKwqpSG5SQrSoOuWKUg4= github.com/smartystreets/assertions v0.0.0-20160201214316-443d812296a8/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c h1:RlFdHEfqB3LI6Q8YfB1EQPO2+kED+w9TdVaMwrs6VPo= github.com/smartystreets/goconvey v1.6.1-0.20160119221636-995f5b2e021c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -288,6 +403,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tinylib/msgp v1.1.3 h1:3giwAkmtaEDLSV0MdO1lDLuPgklgPzmk8H9+So2BVfA= +github.com/tinylib/msgp v1.1.3/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4= +github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= github.com/vitalif/fusego v0.0.0-20220517201526-003d54287c05 h1:SDeqAo7ZTpt0zMzIepDg7y7Bd+C7FhuhMGJMX5euwPI= @@ -300,6 +423,9 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -308,14 +434,22 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -360,6 +494,8 @@ golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -395,6 +531,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 h1:a8jGStKg0XqKDlKqjLrXn0ioF5MH36pT7Z0BRTqLhbk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -409,6 +546,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1 h1:x622Z2o4hgCr/4CiKWc51jHVKaWdtVpBNmEI8wI9Qns= golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -424,8 +562,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -435,6 +575,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -451,6 +592,7 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= @@ -459,11 +601,14 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -471,6 +616,8 @@ golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -499,6 +646,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -508,6 +656,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -530,6 +679,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -538,11 +688,14 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -569,6 +722,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.49.0 h1:gjIBDxlTG7vnzMmEnYwTnvLTF8Rjzo+ETCgEX1YZ/fY= @@ -604,6 +758,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -640,6 +795,7 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -671,11 +827,18 @@ gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/backend.go b/internal/backend.go index b58e0c91..8121e87a 100644 --- a/internal/backend.go +++ b/internal/backend.go @@ -27,7 +27,7 @@ import ( ) type Capabilities struct { - MaxMultipartSize uint64 + MaxMultipartSize uint64 // indicates that the blob store has native support for directories DirBlob bool Name string @@ -44,7 +44,7 @@ type BlobItemOutput struct { Size uint64 StorageClass *string // may be nil in list responses for backends that don't return metadata in listings - Metadata map[string]*string + Metadata map[string]*string } type HeadBlobOutput struct { @@ -179,7 +179,7 @@ type MultipartBlobAddInput struct { type MultipartBlobAddOutput struct { RequestId string - PartId *string + PartId *string } type MultipartBlobCopyInput struct { @@ -192,7 +192,7 @@ type MultipartBlobCopyInput struct { type MultipartBlobCopyOutput struct { RequestId string - PartId *string + PartId *string } type MultipartBlobCommitOutput struct { @@ -228,6 +228,14 @@ type MakeBucketOutput struct { RequestId string } +type GetBucketUsageInput struct { +} + +type GetBucketUsageOutput struct { + Size uint64 + RequestId string +} + /// Implementations of all the functions here are expected to be /// concurrency-safe, except for /// @@ -256,6 +264,7 @@ type StorageBackend interface { MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) + GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) Delegate() interface{} } @@ -424,6 +433,11 @@ func (s *StorageBackendInitWrapper) MakeBucket(param *MakeBucketInput) (*MakeBuc return s.StorageBackend.MakeBucket(param) } +func (s *StorageBackendInitWrapper) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + s.Init("") + return s.StorageBackend.GetBucketUsage(param) +} + type StorageBackendInitError struct { error cap Capabilities @@ -547,3 +561,7 @@ func (e StorageBackendInitError) RemoveBucket(param *RemoveBucketInput) (*Remove func (e StorageBackendInitError) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { return nil, e } + +func (e StorageBackendInitError) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return nil, e +} diff --git a/internal/backend_adlv1.go b/internal/backend_adlv1.go index 5e4e33d1..fbf61f9e 100644 --- a/internal/backend_adlv1.go +++ b/internal/backend_adlv1.go @@ -152,8 +152,8 @@ func NewADLv1(bucket string, flags *FlagStorage, config *ADLv1Config) (*ADLv1, e bucket: bucket, cap: Capabilities{ //NoParallelMultipart: true, - DirBlob: true, - Name: "adl", + DirBlob: true, + Name: "adl", // ADLv1 fails with 404 if we upload data // larger than 30000000 bytes (28.6MB) (28MB // also failed in at one point, but as of @@ -701,3 +701,7 @@ func (b *ADLv1) mkdir(dir string) error { } return nil } + +func (b *ADLv1) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return &GetBucketUsageOutput{Size: 0}, nil +} diff --git a/internal/backend_adlv2.go b/internal/backend_adlv2.go index 85fdfbf4..053e9db5 100644 --- a/internal/backend_adlv2.go +++ b/internal/backend_adlv2.go @@ -207,6 +207,10 @@ func (b *ADLv2) Capabilities() *Capabilities { return &b.cap } +func (b *ADLv2) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return &GetBucketUsageOutput{Size: 0}, nil +} + type ADL2Error struct { adl2.DataLakeStorageError } diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index 8745ad8e..2f0fbe4e 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -222,7 +222,7 @@ func (b *AZBlob) refreshToken() (*azblob.ContainerURL, error) { // our token totally expired, renew inline before using it b.mu.Unlock() b.tokenRenewGate <- 1 - defer func() { <- b.tokenRenewGate } () + defer func() { <-b.tokenRenewGate }() b.mu.Lock() // check again, because in the mean time maybe it's renewed @@ -248,7 +248,7 @@ func (b *AZBlob) refreshToken() (*azblob.ContainerURL, error) { if err != nil { azbLog.Errorf("Unable to refresh token: %v", err) } - <- b.tokenRenewGate + <-b.tokenRenewGate }() // if we cannot renew token, treat it as a @@ -665,7 +665,7 @@ func (b *AZBlob) DeleteBlobs(param *DeleteBlobsInput) (ret *DeleteBlobsOutput, d go func(key string) { defer func() { - <- SmallActionsGate + <-SmallActionsGate wg.Done() }() @@ -947,3 +947,7 @@ func (b *AZBlob) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { } return &MakeBucketOutput{}, nil } + +func (b *AZBlob) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return &GetBucketUsageOutput{Size: 0}, nil +} diff --git a/internal/backend_gcs3.go b/internal/backend_gcs3.go index b5308b1b..5485593b 100644 --- a/internal/backend_gcs3.go +++ b/internal/backend_gcs3.go @@ -23,15 +23,15 @@ import ( "sync" "syscall" - "github.com/jacobsa/fuse" "cloud.google.com/go/storage" + "github.com/jacobsa/fuse" "google.golang.org/api/iterator" ) // GCS variant of S3 type GCS3 struct { *S3Backend - gcs *storage.Client + gcs *storage.Client jsonCredFile string } @@ -103,12 +103,12 @@ func (s *GCS3) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { }) } else { items = append(items, BlobItemOutput{ - Key: &attrs.Name, - ETag: &attrs.Etag, + Key: &attrs.Name, + ETag: &attrs.Etag, LastModified: &attrs.Updated, - Size: uint64(attrs.Size), + Size: uint64(attrs.Size), StorageClass: &attrs.StorageClass, - Metadata: PMetadata(attrs.Metadata), + Metadata: PMetadata(attrs.Metadata), }) } n++ @@ -156,3 +156,7 @@ func (s *GCS3) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) func (s *GCS3) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error) { return nil, syscall.ENOSYS } + +func (s *GCS3) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return &GetBucketUsageOutput{Size: 0}, nil +} diff --git a/internal/backend_minio.go b/internal/backend_minio.go new file mode 100644 index 00000000..54b16295 --- /dev/null +++ b/internal/backend_minio.go @@ -0,0 +1,68 @@ +// Copyright 2019 Ka-Hing Cheung +// Copyright 2021 Yandex LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/url" + + "github.com/minio/madmin-go" + . "github.com/yandex-cloud/geesefs/api/common" +) + +type BucketsUsage struct { + BucketsSizes map[string]uint64 +} + +type MinioBackend struct { + *S3Backend +} + +func NewMinio(bucket string, flags *FlagStorage, config *S3Config) (*MinioBackend, error) { + s3Backend, err := NewS3(bucket, flags, config) + if err != nil { + return nil, err + } + s3Backend.Capabilities().Name = "minio" + s := &MinioBackend{S3Backend: s3Backend} + return s, nil +} + +func (s *MinioBackend) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + value, err := s.Config.Credentials.Get() + if err != nil { + return nil, err + } + + endpointURL, _ := url.Parse(s.Endpoint) + madminClient, err := madmin.New(endpointURL.Host, value.AccessKeyID, value.SecretAccessKey, endpointURL.Scheme == "https") + if err != nil { + return nil, err + } + resp, err := madminClient.ExecuteMethod(context.Background(), "GET", madmin.RequestData{RelPath: "/v3/datausageinfo"}) + if err != nil { + return nil, err + } + response, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var result BucketsUsage + json.Unmarshal(response, &result) + return &GetBucketUsageOutput{Size: result.BucketsSizes[s.Bucket()]}, nil +} diff --git a/internal/backend_s3.go b/internal/backend_s3.go index c42909e1..79d28b19 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -55,15 +55,15 @@ type S3Backend struct { gcs bool v2Signer bool - iam bool - iamToken atomic.Value + iam bool + iamToken atomic.Value iamTokenExpiration time.Time - iamRefreshTimer *time.Timer + iamRefreshTimer *time.Timer } func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, error) { if config.MultipartCopyThreshold == 0 { - config.MultipartCopyThreshold = 128*1024*1024 + config.MultipartCopyThreshold = 128 * 1024 * 1024 } awsConfig, err := config.ToAwsConfig(flags) if err != nil { @@ -100,8 +100,8 @@ func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, err } type IAMCredResponse struct { - Code string - Token string + Code string + Token string Expiration time.Time } @@ -128,11 +128,11 @@ func (s *S3Backend) TryIAM() error { s.iam = true s.iamToken.Store(creds.Token) s.iamTokenExpiration = creds.Expiration - ttl := s.iamTokenExpiration.Sub(time.Now().Add(5*time.Minute)) + ttl := s.iamTokenExpiration.Sub(time.Now().Add(5 * time.Minute)) if ttl < 0 { ttl = s.iamTokenExpiration.Sub(time.Now()) if ttl >= 30*time.Second { - ttl = 30*time.Second + ttl = 30 * time.Second } } s.iamRefreshTimer = time.AfterFunc(ttl, func() { @@ -997,7 +997,7 @@ func (s *S3Backend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBl return &MultipartBlobAddOutput{ RequestId: s.getRequestId(req), - PartId: resp.ETag, + PartId: resp.ETag, }, nil } @@ -1006,7 +1006,7 @@ func (s *S3Backend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*Multipart Bucket: &s.bucket, Key: param.Commit.Key, PartNumber: aws.Int64(int64(param.PartNumber)), - CopySource: aws.String(pathEscape(s.bucket+"/"+param.CopySource)), + CopySource: aws.String(pathEscape(s.bucket + "/" + param.CopySource)), UploadId: param.Commit.UploadId, } if param.Size != 0 { @@ -1028,7 +1028,7 @@ func (s *S3Backend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*Multipart return &MultipartBlobCopyOutput{ RequestId: s.getRequestId(req), - PartId: resp.CopyPartResult.ETag, + PartId: resp.CopyPartResult.ETag, }, nil } @@ -1172,3 +1172,7 @@ func (s *S3Backend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error func (s *S3Backend) Delegate() interface{} { return s } + +func (s *S3Backend) GetBucketUsage(param *GetBucketUsageInput) (*GetBucketUsageOutput, error) { + return &GetBucketUsageOutput{Size: 0}, nil +} diff --git a/internal/flags.go b/internal/flags.go index 35bb3af9..4c94ad97 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -90,13 +90,31 @@ func NewApp() (app *cli.App) { // File system ///////////////////////// + cli.StringFlag{ + Name: "provider", + Value: "unknow", + Usage: "The specific provider of the object storage service.", + }, + + cli.Uint64Flag{ + Name: "capacity", + Value: 1 * 1024 * 1024 * 1024 * 1024 * 1024, + Usage: "Limit the maximum capacity quota of mounted files.", + }, + + cli.Uint64Flag{ + Name: "disk-usage-interval", + Value: 60, + Usage: "Time interval for statistics of disk usage, in seconds.", + }, + cli.StringSliceFlag{ Name: "o", Usage: "Additional system-specific mount options. Be careful!", }, cli.StringFlag{ - Name: "cache", + Name: "cache", Usage: "Directory to use for data cache. (default: off)", }, @@ -286,20 +304,20 @@ func NewApp() (app *cli.App) { cli.IntFlag{ Name: "max-parallel-parts", Value: 8, - Usage: "How much parallel requests out of the total number can be used for large part uploads."+ + Usage: "How much parallel requests out of the total number can be used for large part uploads." + " Large parts take more bandwidth so they usually require less parallelism (default: 8)", }, cli.IntFlag{ Name: "max-parallel-copy", Value: 16, - Usage: "How much parallel unmodified part copy requests should be used."+ + Usage: "How much parallel unmodified part copy requests should be used." + " This limit is separate from max-flushers (default: 16)", }, cli.IntFlag{ Name: "read-ahead", - Value: 5*1024, + Value: 5 * 1024, Usage: "How much data in KB should be pre-loaded with every read by default (default: 5 MB)", }, @@ -323,19 +341,19 @@ func NewApp() (app *cli.App) { cli.IntFlag{ Name: "large-read-cutoff", - Value: 20*1024, + Value: 20 * 1024, Usage: "Amount of linear read in KB after which the \"large\" readahead should be triggered (default: 20 MB)", }, cli.IntFlag{ Name: "read-ahead-large", - Value: 100*1024, + Value: 100 * 1024, Usage: "Larger readahead size in KB to be used when long linear reads are detected (default: 100 MB)", }, cli.IntFlag{ Name: "read-ahead-parallel", - Value: 20*1024, + Value: 20 * 1024, Usage: "Larger readahead will be triggered in parallel chunks of this size in KB (default: 20 MB)", }, @@ -356,7 +374,7 @@ func NewApp() (app *cli.App) { cli.StringFlag{ Name: "part-sizes", Value: "5:1000,25:1000,125", - Usage: "Part sizes in MB. Total part count is always 10000 in S3."+ + Usage: "Part sizes in MB. Total part count is always 10000 in S3." + " Default is 1000 5 MB parts, then 1000 25 MB parts" + " and then 125 MB for the rest of parts", }, @@ -365,7 +383,7 @@ func NewApp() (app *cli.App) { Name: "max-merge-copy", Value: 0, Usage: "If non-zero, allow to compose larger parts up to this number of megabytes" + - " in size from existing unchanged parts when doing server-side part copy."+ + " in size from existing unchanged parts when doing server-side part copy." + " Must be left at 0 for Yandex S3 (default: 0)", }, @@ -473,7 +491,7 @@ func NewApp() (app *cli.App) { Usage: "Mount an S3 bucket locally", HideHelp: true, Writer: os.Stderr, - Flags: append(append(append(append([]cli.Flag{ + Flags: append(append(append(append([]cli.Flag{ cli.BoolFlag{ Name: "help, h", Usage: "Print this help text and exit successfully.", @@ -558,7 +576,7 @@ func parsePartSizes(s string) (result []PartSizeConfig) { if pi < len(partSizes)-1 { panic("Part count may be omitted only for the last interval") } - count = 10000-totalCount + count = 10000 - totalCount } totalCount += count if totalCount > 10000 { @@ -571,7 +589,7 @@ func parsePartSizes(s string) (result []PartSizeConfig) { panic("Maximum part size is 5 GB") } result = append(result, PartSizeConfig{ - PartSize: size*1024*1024, + PartSize: size * 1024 * 1024, PartCount: count, }) } @@ -588,55 +606,58 @@ func PopulateFlags(c *cli.Context) (ret *FlagStorage) { flags := &FlagStorage{ // File system - MountOptions: make(map[string]string), - DirMode: os.FileMode(c.Int("dir-mode")), - FileMode: os.FileMode(c.Int("file-mode")), - Uid: uint32(c.Int("uid")), - Gid: uint32(c.Int("gid")), + MountOptions: make(map[string]string), + DirMode: os.FileMode(c.Int("dir-mode")), + FileMode: os.FileMode(c.Int("file-mode")), + Uid: uint32(c.Int("uid")), + Gid: uint32(c.Int("gid")), // Tuning, - MemoryLimit: uint64(1024*1024*c.Int("memory-limit")), - GCInterval: uint64(1024*1024*c.Int("gc-interval")), - Cheap: c.Bool("cheap"), - ExplicitDir: c.Bool("no-implicit-dir"), - NoDirObject: c.Bool("no-dir-object"), - MaxFlushers: int64(c.Int("max-flushers")), - MaxParallelParts: c.Int("max-parallel-parts"), - MaxParallelCopy: c.Int("max-parallel-copy"), - StatCacheTTL: c.Duration("stat-cache-ttl"), - HTTPTimeout: c.Duration("http-timeout"), - RetryInterval: c.Duration("retry-interval"), - ReadAheadKB: uint64(c.Int("read-ahead")), - SmallReadCount: uint64(c.Int("small-read-count")), - SmallReadCutoffKB: uint64(c.Int("small-read-cutoff")), - ReadAheadSmallKB: uint64(c.Int("read-ahead-small")), - LargeReadCutoffKB: uint64(c.Int("large-read-cutoff")), - ReadAheadLargeKB: uint64(c.Int("read-ahead-large")), - ReadAheadParallelKB: uint64(c.Int("read-ahead-parallel")), - ReadMergeKB: uint64(c.Int("read-merge")), - SinglePartMB: uint64(singlePart), - MaxMergeCopyMB: uint64(c.Int("max-merge-copy")), - IgnoreFsync: c.Bool("ignore-fsync"), - SymlinkAttr: c.String("symlink-attr"), - CachePopularThreshold: int64(c.Int("cache-popular-threshold")), - CacheMaxHits: int64(c.Int("cache-max-hits")), - CacheAgeInterval: int64(c.Int("cache-age-interval")), - CacheAgeDecrement: int64(c.Int("cache-age-decrement")), - CacheToDiskHits: int64(c.Int("cache-to-disk-hits")), - CachePath: c.String("cache"), - MaxDiskCacheFD: int64(c.Int("max-disk-cache-fd")), - CacheFileMode: os.FileMode(c.Int("cache-file-mode")), + MemoryLimit: uint64(1024 * 1024 * c.Int("memory-limit")), + GCInterval: uint64(1024 * 1024 * c.Int("gc-interval")), + Cheap: c.Bool("cheap"), + ExplicitDir: c.Bool("no-implicit-dir"), + NoDirObject: c.Bool("no-dir-object"), + MaxFlushers: int64(c.Int("max-flushers")), + MaxParallelParts: c.Int("max-parallel-parts"), + MaxParallelCopy: c.Int("max-parallel-copy"), + StatCacheTTL: c.Duration("stat-cache-ttl"), + HTTPTimeout: c.Duration("http-timeout"), + RetryInterval: c.Duration("retry-interval"), + ReadAheadKB: uint64(c.Int("read-ahead")), + SmallReadCount: uint64(c.Int("small-read-count")), + SmallReadCutoffKB: uint64(c.Int("small-read-cutoff")), + ReadAheadSmallKB: uint64(c.Int("read-ahead-small")), + LargeReadCutoffKB: uint64(c.Int("large-read-cutoff")), + ReadAheadLargeKB: uint64(c.Int("read-ahead-large")), + ReadAheadParallelKB: uint64(c.Int("read-ahead-parallel")), + ReadMergeKB: uint64(c.Int("read-merge")), + SinglePartMB: uint64(singlePart), + MaxMergeCopyMB: uint64(c.Int("max-merge-copy")), + IgnoreFsync: c.Bool("ignore-fsync"), + SymlinkAttr: c.String("symlink-attr"), + CachePopularThreshold: int64(c.Int("cache-popular-threshold")), + CacheMaxHits: int64(c.Int("cache-max-hits")), + CacheAgeInterval: int64(c.Int("cache-age-interval")), + CacheAgeDecrement: int64(c.Int("cache-age-decrement")), + CacheToDiskHits: int64(c.Int("cache-to-disk-hits")), + CachePath: c.String("cache"), + MaxDiskCacheFD: int64(c.Int("max-disk-cache-fd")), + CacheFileMode: os.FileMode(c.Int("cache-file-mode")), // Common Backend Config - Endpoint: c.String("endpoint"), - UseContentType: c.Bool("use-content-type"), + Provider: c.String("provider"), + Capacity: c.Uint64("capacity"), + DiskUsageInterval: c.Uint64("disk-usage-interval"), + Endpoint: c.String("endpoint"), + UseContentType: c.Bool("use-content-type"), // Debugging, - DebugMain: c.Bool("debug"), - DebugFuse: c.Bool("debug_fuse"), - DebugS3: c.Bool("debug_s3"), - Foreground: c.Bool("f"), - LogFile: c.String("log-file"), + DebugMain: c.Bool("debug"), + DebugFuse: c.Bool("debug_fuse"), + DebugS3: c.Bool("debug_s3"), + Foreground: c.Bool("f"), + LogFile: c.String("log-file"), } flags.PartSizes = parsePartSizes(c.String("part-sizes")) @@ -645,25 +666,25 @@ func PopulateFlags(c *cli.Context) (ret *FlagStorage) { if flags.Backend == nil { flags.Backend = (&S3Config{}).Init() config, _ := flags.Backend.(*S3Config) - config.Region = c.String("region") - config.RegionSet = c.IsSet("region") + config.Region = c.String("region") + config.RegionSet = c.IsSet("region") config.RequesterPays = c.Bool("requester-pays") - config.StorageClass = c.String("storage-class") - config.Profile = c.String("profile") - config.SharedConfig = c.StringSlice("shared-config") - config.UseSSE = c.Bool("sse") - config.UseKMS = c.IsSet("sse-kms") - config.KMSKeyID = c.String("sse-kms") - config.SseC = c.String("sse-c") - config.ACL = c.String("acl") - config.Subdomain = c.Bool("subdomain") - config.NoChecksum = c.Bool("no-checksum") - config.UseIAM = c.Bool("iam") - config.IAMHeader = c.String("iam-header") - config.MultipartAge = c.Duration("multipart-age") + config.StorageClass = c.String("storage-class") + config.Profile = c.String("profile") + config.SharedConfig = c.StringSlice("shared-config") + config.UseSSE = c.Bool("sse") + config.UseKMS = c.IsSet("sse-kms") + config.KMSKeyID = c.String("sse-kms") + config.SseC = c.String("sse-c") + config.ACL = c.String("acl") + config.Subdomain = c.Bool("subdomain") + config.NoChecksum = c.Bool("no-checksum") + config.UseIAM = c.Bool("iam") + config.IAMHeader = c.String("iam-header") + config.MultipartAge = c.Duration("multipart-age") listType := c.String("list-type") - config.ListV1Ext = listType == "ext-v1" - config.ListV2 = listType == "2" + config.ListV1Ext = listType == "ext-v1" + config.ListV2 = listType == "2" config.MultipartCopyThreshold = uint64(c.Int("multipart-copy-threshold")) * 1024 * 1024 diff --git a/internal/goofys.go b/internal/goofys.go index 32f43cd3..59fca5ae 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -36,8 +36,9 @@ import ( "github.com/jacobsa/fuse/fuseops" "github.com/jacobsa/fuse/fuseutil" - "github.com/sirupsen/logrus" "net/http" + + "github.com/sirupsen/logrus" ) // goofys is a Filey System written in Go. All the backend data is @@ -66,8 +67,8 @@ type Goofys struct { // from per-inode locks). Make sure to see the notes on lock ordering above. mu sync.RWMutex - flusherMu sync.Mutex - flusherCond *sync.Cond + flusherMu sync.Mutex + flusherCond *sync.Cond flushPending int32 // The next inode ID to hand out. We assume that this will never overflow, @@ -91,8 +92,8 @@ type Goofys struct { // Inflight changes are tracked to skip them in parallel listings // Required because we don't have guarantees about listing & change ordering inflightListingId int - inflightListings map[int]map[string]bool - inflightChanges map[string]int + inflightListings map[int]map[string]bool + inflightChanges map[string]int nextHandleID fuseops.HandleID dirHandles map[fuseops.HandleID]*DirHandle @@ -100,15 +101,16 @@ type Goofys struct { fileHandles map[fuseops.HandleID]*FileHandle activeFlushers int64 - flushRetrySet int32 - memRecency uint64 + flushRetrySet int32 + memRecency uint64 forgotCnt uint32 - zeroBuf []byte - lfru *LFRU - diskFdMu sync.Mutex - diskFdCond *sync.Cond + zeroBuf []byte + lfru *LFRU + diskUsage uint64 + diskFdMu sync.Mutex + diskFdCond *sync.Cond diskFdCount int64 } @@ -130,6 +132,8 @@ func NewBackend(bucket string, flags *FlagStorage) (cloud StorageBackend, err er } else if config, ok := flags.Backend.(*S3Config); ok { if strings.HasSuffix(flags.Endpoint, "/storage.googleapis.com") { cloud, err = NewGCS3(bucket, flags, config) + } else if flags.Provider == "minio" { + cloud, err = NewMinio(bucket, flags, config) } else { cloud, err = NewS3(bucket, flags, config) } @@ -189,12 +193,12 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, newBackend func(string, *FlagStorage) (StorageBackend, error)) *Goofys { // Set up the basic struct. fs := &Goofys{ - bucket: bucket, - flags: flags, - umask: 0122, - lfru: NewLFRU(flags.CachePopularThreshold, flags.CacheMaxHits, flags.CacheAgeInterval, flags.CacheAgeDecrement), - zeroBuf: make([]byte, 1048576), - inflightChanges: make(map[string]int), + bucket: bucket, + flags: flags, + umask: 0122, + lfru: NewLFRU(flags.CachePopularThreshold, flags.CacheMaxHits, flags.CacheAgeInterval, flags.CacheAgeDecrement), + zeroBuf: make([]byte, 1048576), + inflightChanges: make(map[string]int), inflightListings: make(map[int]map[string]bool), } @@ -222,6 +226,18 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, } _, fs.gcs = cloud.Delegate().(*GCS3) + go func(fs *Goofys, cloud StorageBackend) { + for { + getBucketUsageOutput, err := cloud.GetBucketUsage(&GetBucketUsageInput{}) + if err != nil { + fs.diskUsage = 0 + } else { + fs.diskUsage = getBucketUsageOutput.Size + } + time.Sleep(time.Second * time.Duration(fs.flags.DiskUsageInterval)) + } + }(fs, cloud) + randomObjectName := prefix + (RandStringBytesMaskImprSrc(32)) err = cloud.Init(randomObjectName) if err != nil { @@ -236,7 +252,7 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, Mtime: now, } - fs.bufferPool = NewBufferPool(int64(flags.MemoryLimit), uint64(flags.GCInterval) << 20) + fs.bufferPool = NewBufferPool(int64(flags.MemoryLimit), uint64(flags.GCInterval)<<20) fs.bufferPool.FreeSomeCleanBuffers = func(size int64) (int64, bool) { return fs.FreeSomeCleanBuffers(size) } @@ -271,6 +287,10 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, return fs } +func (fs *Goofys) checkWriteAvailable(size uint64) bool { + return fs.flags.Capacity > (fs.diskUsage + size) +} + // from https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang func RandStringBytesMaskImprSrc(n int) string { const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789" @@ -352,12 +372,12 @@ func (fs *Goofys) FreeSomeCleanBuffers(size int64) (int64, bool) { haveDirty := false // Free at least 5 MB if size < 5*1024*1024 { - size = 5*1024*1024 + size = 5 * 1024 * 1024 } skipRecent := atomic.LoadUint64(&fs.memRecency) // Avoid evicting at least 1/4 of recent memory allocations if skipRecent > fs.flags.MemoryLimit/4 { - skipRecent -= fs.flags.MemoryLimit/4 + skipRecent -= fs.flags.MemoryLimit / 4 } else { skipRecent = 0 } @@ -438,12 +458,12 @@ func (fs *Goofys) FreeSomeCleanBuffers(size int64) (int64, bool) { // A flushed buffer can be removed at a cost of finalizing multipart upload // to read it back later. However it's likely not a problem if we're uploading // a large file because we may never need to read it back. - prev := del-1 + prev := del - 1 if prev < 0 { - prev = i-1 + prev = i - 1 } if prev >= 0 && inode.buffers[prev].state == BUF_FL_CLEARED && - buf.offset == (inode.buffers[prev].offset + inode.buffers[prev].length) { + buf.offset == (inode.buffers[prev].offset+inode.buffers[prev].length) { inode.buffers[prev].length += buf.length if del == -1 { del = i @@ -458,13 +478,13 @@ func (fs *Goofys) FreeSomeCleanBuffers(size int64) (int64, bool) { haveDirty = true } if del >= 0 { - inode.buffers = append(inode.buffers[0 : del], inode.buffers[i : ]...) + inode.buffers = append(inode.buffers[0:del], inode.buffers[i:]...) i = del del = -1 } } if del >= 0 { - inode.buffers = append(inode.buffers[0 : del], inode.buffers[i : ]...) + inode.buffers = append(inode.buffers[0:del], inode.buffers[i:]...) del = -1 } inode.mu.Unlock() @@ -686,16 +706,20 @@ func (fs *Goofys) StatFS( op *fuseops.StatFSOp) (err error) { const BLOCK_SIZE = 4096 - const TOTAL_SPACE = 1 * 1024 * 1024 * 1024 * 1024 * 1024 // 1PB - const TOTAL_BLOCKS = TOTAL_SPACE / BLOCK_SIZE - const INODES = 1 * 1000 * 1000 * 1000 // 1 billion + TOTAL_BLOCKS := fs.flags.Capacity / BLOCK_SIZE + USAGE_BLOCKS := fs.diskUsage / BLOCK_SIZE + op.BlockSize = BLOCK_SIZE op.Blocks = TOTAL_BLOCKS - op.BlocksFree = TOTAL_BLOCKS - op.BlocksAvailable = TOTAL_BLOCKS - op.IoSize = 1 * 1024 * 1024 // 1MB - op.Inodes = INODES - op.InodesFree = INODES + if USAGE_BLOCKS > TOTAL_BLOCKS { + op.BlocksFree = 0 + } else { + op.BlocksFree = TOTAL_BLOCKS - USAGE_BLOCKS + } + op.BlocksAvailable = op.BlocksFree + op.IoSize = 1 * 1024 * 1024 // 1MB + op.Inodes = 1 * 1000 * 1000 * 1000 // 1 billion + op.InodesFree = 1 * 1000 * 1000 * 1000 // 1 billion return } @@ -810,6 +834,9 @@ func (fs *Goofys) RemoveXattr(ctx context.Context, func (fs *Goofys) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) (err error) { + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } fs.mu.RLock() inode := fs.getInodeOrDie(op.Inode) fs.mu.RUnlock() @@ -830,6 +857,9 @@ func (fs *Goofys) SetXattr(ctx context.Context, func (fs *Goofys) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) (err error) { + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } fs.mu.RLock() parent := fs.getInodeOrDie(op.Parent) fs.mu.RUnlock() @@ -1371,7 +1401,9 @@ func (fs *Goofys) ReleaseFileHandle( func (fs *Goofys) CreateFile( ctx context.Context, op *fuseops.CreateFileOp) (err error) { - + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } fs.mu.RLock() parent := fs.getInodeOrDie(op.Parent) fs.mu.RUnlock() @@ -1411,9 +1443,11 @@ func (fs *Goofys) CreateFile( func (fs *Goofys) MkNode( ctx context.Context, op *fuseops.MkNodeOp) (err error) { - - if (op.Mode & os.ModeType) != os.ModeDir && - (op.Mode & os.ModeType) != 0 { + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } + if (op.Mode&os.ModeType) != os.ModeDir && + (op.Mode&os.ModeType) != 0 { // Special files are not supported yet return syscall.ENOTSUP } @@ -1451,6 +1485,9 @@ func (fs *Goofys) MkNode( func (fs *Goofys) MkDir( ctx context.Context, op *fuseops.MkDirOp) (err error) { + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } fs.mu.RLock() parent := fs.getInodeOrDie(op.Parent) @@ -1498,7 +1535,9 @@ func (fs *Goofys) RmDir( func (fs *Goofys) SetInodeAttributes( ctx context.Context, op *fuseops.SetInodeAttributesOp) (err error) { - + if !fs.checkWriteAvailable(0) { + return syscall.ENOSPC + } fs.mu.RLock() inode := fs.getInodeOrDie(op.Inode) fs.mu.RUnlock() @@ -1538,7 +1577,9 @@ func (fs *Goofys) SetInodeAttributes( func (fs *Goofys) WriteFile( ctx context.Context, op *fuseops.WriteFileOp) (err error) { - + if !fs.checkWriteAvailable(uint64(op.Offset)) { + return syscall.ENOSPC + } fs.mu.RLock() fh, ok := fs.fileHandles[op.Handle]