Refurbish the grpc and http plugins for eos (cs3org#4185)

* For GRPC to EOS use the app context, not the passed one

* Refresh compiling the eosgrpc proto

* eos_grpc: small fixes

* Add teststorageperf reva shell command

* Refurbish the grpc and http plugins for eos

* Add README for how to compile the grpc intf to eos

* Add test self-referential release note

* Modify test self-referential release note

* Modify test self-referential release note

* Cosmetics

* NSRequests: use the background context to avoid grpc ugly bug

* Fix lint warning

* Close the body of an http get when not needed anymore

* Fix lint warning

* Set cmd timeout to 15s (allows uploading larger files to loaded servers)

* eosgrpc: use a clean context derived from the original one

* Makefile: use gaia to build revad

* Fix lint warning

* Add new target cernbox-revad, using gaia to build revad

* go lint-fix

* Update Makefile

Co-authored-by: Gianmaria Del Monte <[email protected]>

---------

Co-authored-by: Fabrizio Furano <[email protected]>
Co-authored-by: Gianmaria Del Monte <[email protected]>

Makefile

@@ -41,6 +41,13 @@ BUILD_FLAGS	= -X github.com/cs3org/reva/cmd/revad.gitCommit=$(GIT_COMMIT) -X git
 revad:
 	go build -ldflags "-extldflags=-static $(BUILD_FLAGS)" -o ./cmd/revad/revad ./cmd/revad/main
 
+.PHONY: gaia
+gaia:
+	go install github.com/cs3org/gaia@latest
+
+.PHONY: cernbox-revad
+cernbox-revad: gaia
+	gaia build --with github.com/cernbox/reva-plugins --with github.com/cs3org/reva=$(shell pwd) -o ./cmd/revad/revad
 .PHONY: revad-ceph
 revad-ceph:
 	go build -ldflags "$(BUILD_FLAGS)" -tags ceph -o ./cmd/revad/revad ./cmd/revad/main

changelog/unreleased/grpc-http-refurbish.md

@@ -0,0 +1,5 @@
+Enhancement: Refurbish the grpc and https plugins for eos
+
+This enhancement refurbishes the grpc and https plugins for eos
+
+https://github.com/cs3org/reva/pull/4185

cmd/reva/arguments.go

@@ -217,7 +217,7 @@ func executeCommand(cmd *command, args ...string) (bytes.Buffer, error) {
 		if err != nil {
 			return b, err
 		}
-	case <-time.After(500 * time.Millisecond):
+	case <-time.After(15000 * time.Millisecond):
 		return b, errors.New("command timed out")
 	}
 	return b, nil

cmd/reva/main.go

@@ -93,6 +93,7 @@ var (
 		getlockCommand(),
 		unlockCommand(),
 		helpCommand(),
+		testCommand(),
 	}
 )
 

cmd/reva/test.go

@@ -0,0 +1,99 @@
+// Copyright 2018-2023 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"time"
+)
+
+var testCommand = func() *command {
+	cmd := newCommand("teststorageperf")
+	cmd.Description = func() string {
+		return "Little performance test: upload/download/remove 1000 files into the directory /home/perftest. The source file is /tmp/1MFile"
+	}
+	cmd.Action = func(w ...io.Writer) error {
+
+		start := time.Now()
+
+		b, err := executeCommand(mkdirCommand(), "/home/testperf")
+
+		if err != nil {
+			fmt.Println("Error doing mkdir: ", b, err)
+			return nil
+		}
+
+		elapsedmkdir := time.Since(start)
+
+		start = time.Now()
+
+		for i := 0; i < 1000; i++ {
+
+			b, err := executeCommand(uploadCommand(), "-protocol", "simple", "/tmp/1MFile", "/home/testperf/file-"+strconv.FormatInt(int64(i), 10))
+
+			if err != nil {
+				fmt.Printf("Error uploading file %d\n", i)
+				fmt.Println("Err:", b, err)
+				return nil
+			}
+		}
+
+		elapsedupload := time.Since(start)
+
+		start = time.Now()
+
+		for i := 0; i < 1000; i++ {
+
+			b, err := executeCommand(downloadCommand(), "/home/testperf/file-"+strconv.FormatInt(int64(i), 10), "/tmp/1Mdeleteme")
+
+			if err != nil {
+				fmt.Printf("Error downloading file %d\n", i)
+				fmt.Println("Err:", b, err)
+				return nil
+			}
+		}
+
+		elapseddownload := time.Since(start)
+
+		start = time.Now()
+
+		for i := 0; i < 1000; i++ {
+
+			b, err := executeCommand(rmCommand(), "/home/testperf/file-"+strconv.FormatInt(int64(i), 10))
+
+			if err != nil {
+				fmt.Printf("Error removing file %d\n", i)
+				fmt.Println("Err:", b, err)
+				return nil
+			}
+		}
+
+		elapsedrm := time.Since(start)
+
+		fmt.Printf("mkdir took %s \n", elapsedmkdir)
+		fmt.Printf("upload took %s \n", elapsedupload)
+		fmt.Printf("download took %s \n", elapseddownload)
+		fmt.Printf("rm took %s \n", elapsedrm)
+
+		return nil
+	}
+	return cmd
+}

pkg/appctx/cleanctx.go

@@ -0,0 +1,51 @@
+// Copyright 2018-2023 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package appctx
+
+import (
+	"context"
+	"time"
+)
+
+type cleanCtx struct {
+	ctx context.Context
+}
+
+// ContextGetClean returns a new, clean context derived by the given one.
+func ContextGetClean(ctx context.Context) context.Context {
+	return cleanCtx{
+		ctx: ctx,
+	}
+}
+
+func (c cleanCtx) Deadline() (time.Time, bool) {
+	return c.ctx.Deadline()
+}
+
+func (c cleanCtx) Done() <-chan struct{} {
+	return c.ctx.Done()
+}
+
+func (c cleanCtx) Err() error {
+	return c.ctx.Err()
+}
+
+func (c cleanCtx) Value(key any) any {
+	return nil
+}

pkg/eosclient/eosgrpc/eos_grpc/README.protoc

@@ -0,0 +1,10 @@
+To compile the eos binding into go code:
+
+protoc --go_out=. ./Rpc.proto
+protoc ./Rpc.proto --go_out=plugins=grpc:.
+
+
+NOTE: we have to do this here in order to be sure that a compatible protoc compiler is used.
+Having a CI somewhere compiling this does NOT guarantee that the same golang and protoc will be used,
+and this has created lots of problems in the past
+