Feature error handeling

.github/workflows/e2e-test.yml

@@ -30,15 +30,15 @@ jobs:
           echo AWS_S3_BUCKET='test-bucket' >> .env
           echo S3API_SERVICE_PORT='5005' >> .env
           echo AUTH_LEVEL=0 >> .env
-
-      - name: Substitute secret variables in JSON
-        env:
-          auth_password: ${{ secrets.AUTH_PASSWORD }}
-          KEYCLOAK_SECRET: ${{ secrets.KEYCLOAK_SECRET }}
-        run: |
-          echo "Is auth_password set: $(if [ -z "$auth_password" ]; then echo "No"; else echo "Yes"; fi)"
-          echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)"
-          envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json
+          echo INIT_AUTH=0 >> .env
+      # - name: Substitute secret variables in JSON
+      #   env:
+      #     auth_password: ${{ secrets.AUTH_PASSWORD }}
+      #     KEYCLOAK_SECRET: ${{ secrets.KEYCLOAK_SECRET }}
+      #   run: |
+      #     echo "Is auth_password set: $(if [ -z "$auth_password" ]; then echo "No"; else echo "Yes"; fi)"
+      #     echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)"
+      #     envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json
       - name: Build the docker-compose stack
         run: docker-compose build -d
 

blobstore/blobhandler.go

@@ -1,20 +1,17 @@
 package blobstore
 
 import (
+	"errors"
 	"fmt"
-	"net/http"
 	"os"
 	"strconv"
-	"strings"
 	"sync"
 
 	"github.com/Dewberry/s3api/auth"
-	envcheck "github.com/Dewberry/s3api/env-checker"
+	"github.com/Dewberry/s3api/configberry"
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/labstack/echo/v4"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -56,13 +53,13 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 	}
 
 	if authLvl > 0 {
-		if err := envcheck.CheckEnvVariablesExist([]string{"AUTH_LIMITED_WRITER_ROLE"}); err != nil {
-			log.Fatal(err)
+		if err := configberry.CheckEnvVariablesExist([]string{"AUTH_LIMITED_WRITER_ROLE"}); err != nil {
+			return &config, err
 		}
 		config.Config.AuthLevel = authLvl
 		db, err := auth.NewPostgresDB()
 		if err != nil {
-			log.Fatal(err)
+			return &config, err
 		}
 		config.DB = db
 	}
@@ -73,41 +70,43 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 	}
 	s3Mock, err := strconv.Atoi(s3MockStr)
 	if err != nil {
-		log.Fatalf("could not convert S3_MOCK env variable to integer: %v", err)
+		errMsg := fmt.Errorf("could not convert `S3_MOCK` env variable to integer: %v", err)
+		return &config, errMsg
 	}
 	// Check if the S3_MOCK environment variable is set to "true"
 	if s3Mock == 1 {
-		log.Info("Using MinIO")
+		log.Info("using MinIO")
 
 		// Load MinIO credentials from environment
-		creds := newMinioConfig()
+		mc := newMinioConfig()
 
 		// Validate MinIO credentials, check if they are missing or incomplete
 		// if not then the s3api won't start
-		if err := creds.validateMinioConfig(); err != nil {
-			log.Fatalf("MINIO credentials are either not provided or contain missing variables: %s", err.Error())
+		if err := mc.validateMinioConfig(); err != nil {
+			return &config, err
 		}
 
 		// Create a MinIO session and S3 client
-		s3SVC, sess, err := minIOSessionManager(creds)
+		s3SVC, sess, err := mc.minIOSessionManager()
 		if err != nil {
-			log.Fatalf("failed to create MinIO session: %s", err.Error())
+			return &config, err
 		}
 
 		// Configure the BlobHandler with MinIO session and bucket information
-		config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}, S3Mock: true}}
+		config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{mc.Bucket}, S3Mock: true}}
 		// Return the configured BlobHandler
 		return &config, nil
 	}
 
 	// Using AWS S3
 	// Load AWS credentials from the provided .env.json file
-	log.Debug("looking for .env.json")
+	log.Debug("looking for `.env.json`")
 	awsConfig, err := newAWSConfig(envJson)
 
 	// Check if loading AWS credentials from .env.json failed
 	if err != nil {
-		return nil, fmt.Errorf("env.json credentials extraction failed, please check `.env.json.example` for reference on formatting, %s", err.Error())
+		errMsg := fmt.Errorf("`env.json` credentials extraction failed, please check `.env.json.example` for reference on formatting, %s", err.Error())
+		return &config, errMsg
 	}
 
 	//does it contain "*"
@@ -120,21 +119,20 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 	}
 
 	// Load AWS credentials for multiple accounts from .env.json
-	for _, creds := range awsConfig.Accounts {
+	for _, ac := range awsConfig.Accounts {
 		// Create an AWS session and S3 client for each account
-		s3SVC, sess, err := aWSSessionManager(creds)
+		s3SVC, sess, err := ac.aWSSessionManager()
 		if err != nil {
 			errMsg := fmt.Errorf("failed to create AWS session: %s", err.Error())
-			log.Error(errMsg.Error())
-			return nil, errMsg
+			return &config, errMsg
 		}
 
 		S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC}
 		// Retrieve the list of buckets for each account
 		result, err := S3Ctrl.ListBuckets()
 		if err != nil {
-			errMsg := fmt.Errorf("failed to retrieve list of buckets with access key: %s, error: %s", creds.AWS_ACCESS_KEY_ID, err.Error())
-			return nil, errMsg
+			errMsg := fmt.Errorf("failed to retrieve list of buckets with access key: %s, error: %s", ac.AWS_ACCESS_KEY_ID, err.Error())
+			return &config, errMsg
 		}
 
 		var bucketNames []string
@@ -164,63 +162,34 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 		for bucket := range allowedBucketsMap {
 			missingBuckets = append(missingBuckets, bucket)
 		}
-		return nil, fmt.Errorf("some buckets in the allow list were not found: %v", missingBuckets)
+		errMsg := fmt.Errorf("some buckets in the `bucket_allow_list` were not found: %v", missingBuckets)
+		return &config, errMsg
 	}
 
 	// Return the configured BlobHandler
 	return &config, nil
 }
 
-func aWSSessionManager(creds AWSCreds) (*s3.S3, *session.Session, error) {
-	log.Info("Using AWS S3")
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String("us-east-1"),
-		Credentials: credentials.NewStaticCredentials(creds.AWS_ACCESS_KEY_ID, creds.AWS_SECRET_ACCESS_KEY, ""),
+func (s3Ctrl *S3Controller) getBucketRegion(bucketName string) (string, error) {
+	req, output := s3Ctrl.S3Svc.GetBucketLocationRequest(&s3.GetBucketLocationInput{
+		Bucket: aws.String(bucketName),
 	})
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating s3 session: %s", err.Error())
-	}
-	return s3.New(sess), sess, nil
-}
 
-func minIOSessionManager(mc MinioConfig) (*s3.S3, *session.Session, error) {
-	sess, err := session.NewSession(&aws.Config{
-		Endpoint:         aws.String(mc.S3Endpoint),
-		Region:           aws.String("us-east-1"),
-		Credentials:      credentials.NewStaticCredentials(mc.AccessKeyID, mc.SecretAccessKey, ""),
-		S3ForcePathStyle: aws.Bool(true),
-	})
+	err := req.Send()
 	if err != nil {
-		return nil, nil, fmt.Errorf("error connecting to minio session: %s", err.Error())
+		return "", err
 	}
-	log.Info("Using minio to mock s3")
 
-	// Check if the bucket exists
-	s3SVC := s3.New(sess)
-	_, err = s3SVC.HeadBucket(&s3.HeadBucketInput{
-		Bucket: aws.String(mc.Bucket),
-	})
-	if err != nil {
-		// Bucket does not exist, create it
-		_, err := s3SVC.CreateBucket(&s3.CreateBucketInput{
-			Bucket: aws.String(mc.Bucket),
-		})
-		if err != nil {
-			log.Errorf("Error creating bucket: %s", err.Error())
-			return nil, nil, nil
-		}
-		log.Info("Bucket created successfully")
-	} else {
-		log.Info("Bucket already exists")
+	if output.LocationConstraint == nil {
+		return "us-east-1", nil
 	}
 
-	return s3SVC, sess, nil
+	return *output.LocationConstraint, nil
 }
 
 func (bh *BlobHandler) GetController(bucket string) (*S3Controller, error) {
 	if bucket == "" {
-		err := fmt.Errorf("parameter 'bucket' is required")
-		log.Error(err.Error())
+		err := errors.New("parameter `bucket` is required")
 		return nil, err
 	}
 	var s3Ctrl S3Controller
@@ -230,15 +199,15 @@ func (bh *BlobHandler) GetController(bucket string) (*S3Controller, error) {
 				s3Ctrl = bh.S3Controllers[i]
 
 				// Detect the bucket's region
-				region, err := getBucketRegion(bh.S3Controllers[i].S3Svc, b)
+				region, err := s3Ctrl.getBucketRegion(b)
 				if err != nil {
 					log.Errorf("Failed to get region for bucket '%s': %s", b, err.Error())
 					continue
 				}
 				// Check if the region is the same. If not, update the session and client
 				currentRegion := *s3Ctrl.Sess.Config.Region
 				if currentRegion != region {
-					log.Debugf("current region: %s region of bucket: %s, attempting to create a new controller", currentRegion, region)
+					log.Debugf("current region: %s, region of bucket: %s, attempting to create a new controller", currentRegion, region)
 
 					newSession, err := session.NewSession(&aws.Config{
 						Region:      aws.String(region),
@@ -259,98 +228,6 @@ func (bh *BlobHandler) GetController(bucket string) (*S3Controller, error) {
 			}
 		}
 	}
-	return &s3Ctrl, fmt.Errorf("bucket '%s' not found", bucket)
-}
-
-func getBucketRegion(S3Svc *s3.S3, bucketName string) (string, error) {
-	req, output := S3Svc.GetBucketLocationRequest(&s3.GetBucketLocationInput{
-		Bucket: aws.String(bucketName),
-	})
-
-	err := req.Send()
-	if err != nil {
-		return "", err
-	}
-
-	if output.LocationConstraint == nil {
-		return "us-east-1", nil
-	}
-
-	return *output.LocationConstraint, nil
-}
-
-func (bh *BlobHandler) Ping(c echo.Context) error {
-	return c.JSON(http.StatusOK, "connection without Auth is healthy")
-}
-
-func (bh *BlobHandler) PingWithAuth(c echo.Context) error {
-	// Perform a HeadBucket operation to check the health of the S3 connection
-	bucketHealth := make(map[string]string)
-	var valid string
-
-	for _, s3Ctrl := range bh.S3Controllers {
-		for _, b := range s3Ctrl.Buckets {
-			_, err := s3Ctrl.S3Svc.HeadBucket(&s3.HeadBucketInput{
-				Bucket: aws.String(b),
-			})
-			if err != nil {
-				valid = "unhealthy"
-			} else {
-				valid = "healthy"
-			}
-			log.Debugf("Ping operation preformed succesfully, connection to `%s` is %s", b, valid)
-
-			bucketHealth[b] = valid
-			print(b, valid)
-		}
-	}
-
-	return c.JSON(http.StatusOK, bucketHealth)
-}
-
-func (bh *BlobHandler) GetS3ReadPermissions(c echo.Context, bucket string) ([]string, bool, int, error) {
-	permissions, fullAccess, err := bh.GetUserS3ReadListPermission(c, bucket)
-	if err != nil {
-		//TEMP solution before error library is implimented and string check ups become redundant
-		httpStatus := http.StatusInternalServerError
-		if strings.Contains(err.Error(), "this endpoint requires authentication information that is unavailable when authorization is disabled.") {
-			httpStatus = http.StatusForbidden
-		}
-		return nil, false, httpStatus, fmt.Errorf("error fetching user permissions: %s", err.Error())
-	}
-	if !fullAccess && len(permissions) == 0 {
-		return nil, false, http.StatusForbidden, fmt.Errorf("user does not have permission to read the %s bucket", bucket)
-	}
-	return permissions, fullAccess, http.StatusOK, nil
-}
-
-func (bh *BlobHandler) HandleCheckS3UserPermission(c echo.Context) error {
-	if bh.Config.AuthLevel == 0 {
-		log.Info("Checked user permissions successfully")
-		return c.JSON(http.StatusOK, true)
-	}
-	initAuth := os.Getenv("INIT_AUTH")
-	if initAuth == "0" {
-		errMsg := fmt.Errorf("this endpoint requires authentication information that is unavailable when authorization is disabled. Please enable authorization to use this functionality")
-		log.Error(errMsg.Error())
-		return c.JSON(http.StatusForbidden, errMsg.Error())
-	}
-	prefix := c.QueryParam("prefix")
-	bucket := c.QueryParam("bucket")
-	operation := c.QueryParam("operation")
-	claims, ok := c.Get("claims").(*auth.Claims)
-	if !ok {
-		errMsg := fmt.Errorf("could not get claims from request context")
-		log.Error(errMsg.Error())
-		return c.JSON(http.StatusInternalServerError, errMsg.Error())
-	}
-	userEmail := claims.Email
-	if operation == "" || prefix == "" || bucket == "" {
-		errMsg := fmt.Errorf("`prefix`,  `operation` and 'bucket are required params")
-		log.Error(errMsg.Error())
-		return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
-	}
-	isAllowed := bh.DB.CheckUserPermission(userEmail, bucket, prefix, []string{operation})
-	log.Info("Checked user permissions successfully")
-	return c.JSON(http.StatusOK, isAllowed)
+	errMsg := fmt.Errorf("`bucket` '%s' not found", bucket)
+	return &s3Ctrl, errMsg
 }