diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..5534ad0 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,27 @@ +name: Go package + +on: + push: + branches: + - master + pull_request: + +jobs: + ci: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: stable + + - name: Test + run: go test -v ./... + + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dc30e30..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: go -go: - - 1.x -notifications: - email: false - diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..19e0a33 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013 Jehiah Czebotar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index a078c55..56b4fbf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # GoMRJob -[![Build Status](https://secure.travis-ci.org/jehiah/gomrjob.png?branch=master)](http://travis-ci.org/jehiah/gomrjob) [![GoDoc](https://godoc.org/github.com/jehiah/gomrjob?status.svg)](https://godoc.org/github.com/jehiah/gomrjob) [![GitHub release](https://img.shields.io/github/release/jehiah/gomrjob.svg)](https://github.com/jehiah/gomrjob/releases/latest) +![Build Status](https://github.com/jehiah/gomrjob/actions/workflows/ci.yml/badge.svg?branch=master) +[![Docs](https://pkg.go.dev/badge/github.com/jehiah/gomrjob.svg)](http://pkg.go.dev/github.com/jehiah/gomrjob) +[![GitHub release](https://img.shields.io/github/release/jehiah/gomrjob.svg)](https://github.com/jehiah/gomrjob/releases/latest) A Go framework for running Map Reduce Jobs on Hadoop. diff --git a/hdfs/hadoop.go b/hdfs/hadoop.go index 6154d54..6698226 100644 --- a/hdfs/hadoop.go +++ b/hdfs/hadoop.go @@ -42,7 +42,7 @@ func StreamingJar() (string, error) { if hadoopHome == "" { return "", errors.New("env HADOOP_HOME not set") } - p := regexp.MustCompile("^hadoop.*streaming.*\\.jar$") + p := regexp.MustCompile(`^hadoop.*streaming.*\.jar$`) w := func(pathString string, info os.FileInfo, err error) error { if p.FindString(path.Base(pathString)) != "" { streamingJarPath = pathString @@ -50,7 +50,10 @@ func StreamingJar() (string, error) { } return nil } - filepath.Walk(hadoopHome, w) + err := filepath.Walk(hadoopHome, w) + if err != nil { + return streamingJarPath, err + } if streamingJarPath == "" { return "", errors.New("no streaming.jar found") } diff --git a/hdfs/job.go b/hdfs/job.go index cb1c93d..0ac6ae5 100644 --- a/hdfs/job.go +++ b/hdfs/job.go @@ -38,9 +38,7 @@ func absolutePath(path, proto string) string { if proto == "" { proto = "hdfs:///" } - if strings.HasPrefix(path, "/") { - path = path[1:] - } + path = strings.TrimPrefix(path, "/") return proto + path } diff --git a/internal/gcloud/gcloud.go b/internal/gcloud/gcloud.go index 4032870..29c93fc 100644 --- a/internal/gcloud/gcloud.go +++ b/internal/gcloud/gcloud.go @@ -1,10 +1,11 @@ package gcloud import ( - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - "io/ioutil" + "context" "net/http" + "os" + + "golang.org/x/oauth2/google" ) const ( @@ -16,7 +17,7 @@ const ( // This reads a json service account credential file, and returns a http.Client that's bound to an // oauth2 access token for the provided scopes func LoadFromServiceJSON(serviceAccountPath string, scope ...string) (*http.Client, error) { - data, err := ioutil.ReadFile(serviceAccountPath) + data, err := os.ReadFile(serviceAccountPath) if err != nil { return nil, err } @@ -24,5 +25,5 @@ func LoadFromServiceJSON(serviceAccountPath string, scope ...string) (*http.Clie if err != nil { return nil, err } - return conf.Client(oauth2.NoContext), nil + return conf.Client(context.Background()), nil } diff --git a/mrproto/protocol.go b/mrproto/protocol.go index 915028f..d2ee1d8 100644 --- a/mrproto/protocol.go +++ b/mrproto/protocol.go @@ -242,10 +242,10 @@ func JsonInternalOutputProtocol(writer io.Writer) (*sync.WaitGroup, chan<- KeyVa log.Printf("%s - failed encoding %v", err, kv.Value) continue } - w.Write(kBytes) - w.Write(tab) - w.Write(vBytes) - w.Write(newline) + w.Write(kBytes) // nolint:errcheck + w.Write(tab) // nolint:errcheck + w.Write(vBytes) // nolint:errcheck + w.Write(newline) // nolint:errcheck } w.Flush() wg.Done() @@ -275,10 +275,10 @@ func RawJsonInternalOutputProtocol(writer io.Writer) (*sync.WaitGroup, chan<- Ke log.Printf("%s - failed encoding %v", err, kv.Value) continue } - w.Write(kBytes) - w.Write(tab) - w.Write(vBytes) - w.Write(newline) + w.Write(kBytes) // nolint:errcheck + w.Write(tab) // nolint:errcheck + w.Write(vBytes) // nolint:errcheck + w.Write(newline) // nolint:errcheck } w.Flush() wg.Done() diff --git a/mrproto/protocol_test.go b/mrproto/protocol_test.go index 16b537b..c74d2ca 100644 --- a/mrproto/protocol_test.go +++ b/mrproto/protocol_test.go @@ -57,7 +57,7 @@ func TestRawInternalChanInputProtocol(t *testing.T) { consume := func(r io.Reader) (keys int, values int) { for kv := range RawInternalChanInputProtocol(r) { keys++ - for _ = range kv.Values { + for range kv.Values { values++ } } diff --git a/mrtest/testing.go b/mrtest/testing.go index dbf9e4e..f64ba11 100644 --- a/mrtest/testing.go +++ b/mrtest/testing.go @@ -40,8 +40,14 @@ func sortPhase(in io.Reader, out io.Writer) error { } sort.Sort(sortedData{data}) for _, line := range data { - out.Write(bytes.TrimRight(line, "\n")) - out.Write([]byte("\n")) + _, err := out.Write(bytes.TrimRight(line, "\n")) + if err != nil { + return err + } + _, err = out.Write([]byte("\n")) + if err != nil { + return err + } } return nil } diff --git a/remote_logging.go b/remote_logging.go index dd8e526..94681bf 100644 --- a/remote_logging.go +++ b/remote_logging.go @@ -42,7 +42,7 @@ func startRemoteLogListner() string { continue } log.Printf("accepted remote logging connection from %s", conn.RemoteAddr()) - go io.Copy(os.Stderr, conn) + go io.Copy(os.Stderr, conn) // nolint:errcheck } }() diff --git a/reporter.go b/reporter.go index 8ecc50d..6903e23 100644 --- a/reporter.go +++ b/reporter.go @@ -14,7 +14,7 @@ func Counter(group string, counter string, amount int64) { os.Stderr.Sync() } -// reporter:status: +// reporter:status: func Status(message string) { fmt.Fprintf(os.Stderr, "reporter:status:%s\n", message) os.Stderr.Sync() diff --git a/runner.go b/runner.go index a7cab3d..bc3527f 100644 --- a/runner.go +++ b/runner.go @@ -250,7 +250,7 @@ func (r *Runner) Stage() string { // Run is the program entry point from main() // -// When executed directly (--stage='') uploads loads the executibile +// When executed directly (--stage=”) uploads loads the executibile // and submits mapreduce jobs for each stage of the program func (r *Runner) Run() error { if *step >= len(r.Steps) {