Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implemented Artifactory upload and download handling #30

Open
wants to merge 53 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
1675df4
Implemented Artifactory upload and download handling
eeasaa01 Nov 5, 2024
0a01f2e
Fixed authentication error and adjusted log level to debug for succes…
eeasaa01 Nov 5, 2024
412a3aa
Conditional ARTIFACTORY_TOKEN check and removed spurious comment
eeasaa01 Nov 5, 2024
825c3ea
Restored COMMENT_PREFIX and improved error logging for resource creat…
eeasaa01 Nov 5, 2024
d6229c9
Fixed Artifactory integration and addressed PR feedback
eeasaa01 Nov 6, 2024
0ea343f
Implemented requested changes and added tests for Artifactory integra…
eeasaa01 Nov 8, 2024
082c98b
Added comprehensive tests for Artifactory cache functionality
eeasaa01 Nov 8, 2024
a484aa3
Updating delete function to also delete artifactory cache
amdurh05 Nov 11, 2024
61ec037
Implemented Artifactory upload and download handling
eeasaa01 Nov 5, 2024
e98d7af
Fixed authentication error and adjusted log level to debug for succes…
eeasaa01 Nov 5, 2024
503e529
Conditional ARTIFACTORY_TOKEN check and removed spurious comment
eeasaa01 Nov 5, 2024
f5f034e
Restored COMMENT_PREFIX and improved error logging for resource creat…
eeasaa01 Nov 5, 2024
410b85b
Fixed Artifactory integration and addressed PR feedback
eeasaa01 Nov 6, 2024
4033f12
Implemented requested changes and added tests for Artifactory integra…
eeasaa01 Nov 8, 2024
c389fb2
Added comprehensive tests for Artifactory cache functionality
eeasaa01 Nov 8, 2024
2ab9448
Cleanup: Remove binary files and add gitignore
eeasaa01 Nov 12, 2024
2090e45
Fix: Code review feedback and cleanup
eeasaa01 Nov 13, 2024
1ee74b5
Fix: Pass token as parameter to uploadToArtifactory
eeasaa01 Nov 13, 2024
f0676ab
Fix: Prefix NO_CACHE_UPLOAD with GRABIT_
eeasaa01 Nov 13, 2024
5d9a7c8
Fix: Code improvements and cleanup
eeasaa01 Nov 13, 2024
513e420
Fix: Use requests library for cache upload
eeasaa01 Nov 13, 2024
8a41679
Update cmd/add.go with feedback implementation
eeasaa01 Nov 13, 2024
f2e8c42
Update cmd/cache_test.go with cache download tests
eeasaa01 Nov 13, 2024
5ba7cdd
Update log level to warn when failing to upload to cache
eeasaa01 Nov 13, 2024
335070a
Update hash.go and lock.go based on code review
eeasaa01 Nov 13, 2024
a10e728
Update .gitignore
eeasaa01 Nov 13, 2024
d4dcfd7
Merge branch 'feature/artifactory-upload' into feature/artifactory-de…
eeasaa01 Nov 15, 2024
3396a6f
Merge pull request #1 from eeasaa01/feature/artifactory-delete
eeasaa01 Nov 15, 2024
aa45dcb
Delete cmd/6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f768631401…
eeasaa01 Nov 18, 2024
de40802
Update add.go
eeasaa01 Nov 18, 2024
932ceef
Delete cmd/cache_test.go
eeasaa01 Nov 18, 2024
60c174f
Update download.go
eeasaa01 Nov 18, 2024
c1afae0
Delete cmd/test.txt
eeasaa01 Nov 18, 2024
1623787
Update hash.go
eeasaa01 Nov 18, 2024
ee5fafe
Update lock.go
eeasaa01 Nov 18, 2024
5d77339
Update resource.go
eeasaa01 Nov 18, 2024
1acba07
Delete a640986bc257389dd379886fdae6264c8cfa56bc98b71ce3ae3dfbd8ce61dbe3
eeasaa01 Nov 18, 2024
6794f82
Delete openmpi-5.0.5.tar.bz2
eeasaa01 Nov 18, 2024
1273d45
Replace old project files with new ones
eeasaa01 Nov 18, 2024
e5aefe8
Fix typo in filename: dowload_test.go -> download_test.go
eeasaa01 Nov 18, 2024
2b68e57
Fix merge conflict
amdurh05 Nov 19, 2024
f7f3445
readding artifactory add functionality
amdurh05 Nov 21, 2024
f93213b
Updating delete function and changing Artifactory filename to SRI format
amdurh05 Nov 22, 2024
8301d12
Fix bug in AddResource
amdurh05 Nov 22, 2024
534d546
Fix Artifactory download logic and tests for resource cache behavior
eeasaa01 Nov 26, 2024
9344f52
Update resource.go
eeasaa01 Nov 27, 2024
01d5f3e
Update lock.go
eeasaa01 Nov 27, 2024
1f5cf14
Update artifactory_cache_test.go
eeasaa01 Nov 27, 2024
2d597ba
Update lock.go
eeasaa01 Nov 27, 2024
738ec4b
Delete grsbit
eeasaa01 Nov 27, 2024
ba91254
refactor: replace http client with requests library in lock.go
eeasaa01 Nov 27, 2024
fb7c978
fix: update logic in lock.go
eeasaa01 Nov 27, 2024
79d9e43
WIP: Save local changes before pulling
eeasaa01 Nov 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dist/
grabit
grabit.lock
coverage.out
.DS_Store
*.bz2
cmd/test.txt
cmd/6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72
Binary file not shown.
124 changes: 115 additions & 9 deletions cmd/add.go
Original file line number Diff line number Diff line change
@@ -1,54 +1,160 @@
// Copyright (c) 2023 Cisco Systems, Inc. and its affiliates
// All rights reserved.

package cmd

import (
"bytes"
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"

"github.com/cisco-open/grabit/internal"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)

func addAdd(cmd *cobra.Command) {
addCmd := &cobra.Command{
Use: "add",
Use: "add [url]",
Short: "Add new resource",
Args: cobra.MinimumNArgs(1),
RunE: runAdd,
}

addCmd.Flags().String("cache", "", "Artifactory cache URL")
addCmd.Flags().String("algo", internal.RecommendedAlgo, "Integrity algorithm")
addCmd.Flags().String("filename", "", "Target file name to use when downloading the resource")
addCmd.Flags().String("filename", "", "Target file name")
addCmd.Flags().StringArray("tag", []string{}, "Resource tags")

cmd.AddCommand(addCmd)
}

func runAdd(cmd *cobra.Command, args []string) error {
// Get flags
lockFile, err := cmd.Flags().GetString("lock-file")
if err != nil {
return err
}
lock, err := internal.NewLock(lockFile, true)
if err != nil {
return err
}

algo, err := cmd.Flags().GetString("algo")
if err != nil {
return err
}

tags, err := cmd.Flags().GetStringArray("tag")
if err != nil {
return err
}

filename, err := cmd.Flags().GetString("filename")
if err != nil {
return err
}
err = lock.AddResource(args, algo, tags, filename)

cache, err := cmd.Flags().GetString("cache")
if err != nil {
return err
}
err = lock.Save()

// Check for GRABIT_ARTIFACTORY_TOKEN if cache is specified
if cache != "" {
token := os.Getenv("GRABIT_ARTIFACTORY_TOKEN")
if token == "" {
return fmt.Errorf("GRABIT_ARTIFACTORY_TOKEN must be set when using cache")
}
}

// Create or open lock file
lock, err := internal.NewLock(lockFile, true)
if err != nil {
return err
}

// Download file first
ctx := context.Background()
tempFile, err := internal.GetUrltoTempFile(args[0], ctx)
if err != nil {
return fmt.Errorf("failed to download resource: %w", err)
}
defer os.Remove(tempFile)

// Calculate hash for integrity and cache path
hash, err := internal.GetFileHash(tempFile)
if err != nil {
return fmt.Errorf("failed to calculate file hash: %w", err)
}

// Handle cache if specified
var cachePath string
if cache != "" {
// Ensure cache URL ends with a single /
cache = strings.TrimSuffix(cache, "/") + "/"
cachePath = cache + hash

log.Debug().
Str("path", cachePath).
Msg("Uploading to Artifactory cache")

if err := uploadToArtifactory(tempFile, cachePath); err != nil {
log.Debug().
Err(err).
Str("path", cachePath).
Msg("Failed to upload to cache")
// Continue without cache - this is not a fatal error
} else {
log.Debug().
Str("path", cachePath).
Msg("Successfully uploaded to cache")
}
}

// Add resource to lock file
if err := lock.AddResourceWithCache(args, algo, tags, filename, cachePath); err != nil {
return fmt.Errorf("failed to add resource to lock file: %w", err)
}

return lock.Save()
}

func uploadToArtifactory(filePath, cacheUrl string) error {
eeasaa01 marked this conversation as resolved.
Show resolved Hide resolved
token := os.Getenv("GRABIT_ARTIFACTORY_TOKEN")
if token == "" {
return fmt.Errorf("GRABIT_ARTIFACTORY_TOKEN must be set")
}

// Read file content
fileData, err := ioutil.ReadFile(filePath)
if err != nil {
return fmt.Errorf("failed to read file: %w", err)
}

// Create request
req, err := http.NewRequest(http.MethodPut, cacheUrl, bytes.NewReader(fileData))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}

// Set headers
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(fileData)))

// Make request
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to upload to Artifactory: %w", err)
}
defer resp.Body.Close()

// Check response
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
eeasaa01 marked this conversation as resolved.
Show resolved Hide resolved
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("upload failed (status %d): %s", resp.StatusCode, string(body))
}

return nil
}
Loading
Loading