Merge pull request #80 from wtsi-npg/devel

Merge devel to master for release

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/run-tests.yml

@@ -0,0 +1,105 @@
+name: "Unit tests"
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail
+
+    defaults:
+      run:
+        shell: bash -l -e -o pipefail {0}
+
+    env:
+      WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
+      CONDA_TEST_ENVIRONMENT: "testenv"
+
+    strategy:
+      matrix:
+        go: [ "1.16" ]
+        baton: [ "2.1.0" ]
+        experimental: [false]
+        include:
+          - irods: "4.2.7"
+            server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
+            experimental: false
+          - irods: "4.2.8"
+            server_image: "wsinpg/ub-18.04-irods-4.2.8:latest"
+            experimental: true
+
+    services:
+      irods:
+        image: ${{ matrix.server_image }}
+        ports:
+          - 1247:1247
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: "Set up Go ${{ matrix.go }}"
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go }}
+
+    - name: "Initialize Miniconda"
+      run: |
+        echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
+
+    - name: "Configure Miniconda"
+      run: |
+        conda config --prepend pkgs_dirs "$HOME/conda/pkgs"
+        conda config --prepend envs_dirs "$HOME/conda/envs"
+
+        conda config --set auto_update_conda False
+        conda config --prepend channels "$WSI_CONDA_CHANNEL"
+        conda config --append channels conda-forge
+        conda info
+
+    - name: "Install iRODS clients"
+      run: |
+        conda create -y -n "$CONDA_TEST_ENVIRONMENT"
+        conda install -y -n "$CONDA_TEST_ENVIRONMENT" "irods-icommands ==${{ matrix.irods }}"
+        conda install -y -n "$CONDA_TEST_ENVIRONMENT" "baton ==${{ matrix.baton }}"
+
+    - name: "Configure iRODS clients"
+      run: |
+        conda activate "$CONDA_TEST_ENVIRONMENT"
+
+        mkdir -p "$HOME/.irods"
+        cat <<'EOF' > "$HOME/.irods/irods_environment.json"
+        {
+            "irods_host": "localhost",
+            "irods_port": 1247,
+            "irods_user_name": "irods",
+            "irods_zone_name": "testZone",
+            "irods_home": "/testZone/home/irods",
+            "irods_default_resource": "testResc"
+        }
+        EOF
+
+        echo "irods" | script -q -c "iinit" /dev/null
+        ienv
+        ils
+
+        baton-do --version
+
+    - name: "Cache Go modules"
+      uses: actions/cache@v2
+      with:
+        path: |
+          "$HOME/go/pkg/mod"
+          "$HOME/.cache/go-build"
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: "Install test runner"
+      run: |
+        go get github.com/onsi/ginkgo/ginkgo
+        go get github.com/onsi/gomega/...
+
+    - name: "Run tests"
+      run: |
+        conda activate "$CONDA_TEST_ENVIRONMENT"
+        make test

.travis.yml

@@ -6,16 +6,25 @@ services:
 language: go
 
 go:
-  - "1.13"
   - "1.14"
+  - "1.15"
+
+_iRODS_4_2_8: &iRODS_4_2_8 DOCKER_IMAGE="wsinpg/ub-18.04-irods-4.2.8:latest" IRODS_VERSION="==4.2.8" BATON_VERSION=">2.0.1"
 
 env:
   global:
     - GO111MODULE=on
     - WSI_CONDA_CHANNEL="https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
-  matrix:
-    - DOCKER_IMAGE="wsinpg/ub-12.04-irods-4.1:latest" IRODS_VERSION="==4.1.12" BATON_VERSION="==2.0.1"
+  jobs:
     - DOCKER_IMAGE="wsinpg/ub-16.04-irods-4.2.7:latest" IRODS_VERSION="==4.2.7" BATON_VERSION=">2.0.1"
+    - *iRODS_4_2_8
+
+jobs:
+  allow_failures:
+    - go: "1.14"
+      env: *iRODS_4_2_8
+    - go: "1.15"
+      env: *iRODS_4_2_8
 
 cache:
   directories:

CHANGELOG.md

@@ -4,7 +4,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased] - [![Build Status](https://travis-ci.org/kjsanger/valet.svg?branch=devel)](https://travis-ci.org/kjsanger/valet)
+## [Unreleased] - [![Unit tests](https://github.com/wtsi-npg/valet/actions/workflows/run-tests.yml/badge.svg)](https://github.com/wtsi-npg/valet/actions/workflows/run-tests.yml)
+
+### Added
+
+- Add Github Actions test workflow
+- Improve recognition of files to be processed and archived
+- Add support for TSV files.
+- Add support for compressing CSV files.
+- Add new all ONT-specific paths to the exclusion list.
+- Add the /data/laboratory WSI-specific path to the exclusion list.
+- Add iRODS 4.2.8 support
+
+### Removed
+
+- Remove iRODS 4.1.12 support
+
+### Changed
+
+- Relocate repository from github.com/kjsanger to github.com/wtsi-npg
+
+- Bump github.com/wtsi-npg/extendo/v2 from 2.2.0 to 2.4.0
+- Bump github.com/wtsi-npg/logshim from to 1.1.0 to 1.3.0
+- Bump github.com/wtsi-npg/logshim-zerolog from 1.1.0 to 1.3.0
+- Bump github.com/klauspost/pgzip from 1.2.1 to 1.2.5
+- Bump github.com/onsi/ginkgo from 1.12.3 to 1.16.1
+- Bump github.com/onsi/gomega from 1.10.1 to 1.11.0
+- Bump github.com/rs/zerolog from 1.19.1 to 1.21.0
+- Bump github.com/spf13/cobra from 0.0.5 to 0.0.7
 
 ## [1.4.0] - 2020-09-14
 
@@ -16,7 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Bump github.com/kjsanger/extendo from 2.1.0 to 2.2.0
+- Bump github.com/wtsi-npg/extendo from 2.1.0 to 2.2.0
 
 ## [1.3.0] - 2020-06-02
 
@@ -26,7 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Bump github.com/kjsanger/extendo from 2.0.0 to 2.1.0
+- Bump github.com/wtsi-npg/extendo from 2.0.0 to 2.1.0
 - Bump github.com/stretchr/testify from 1.5.1 to 1.6.0
 - Bump github.com/onsi/ginkgo from 1.12.0 to 1.12.2
 
@@ -43,7 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Improve verbose level logging for consistency and information.
 - Add to default paths to ignore in the data root.
 
-- Bump github.com/kjsanger/extendo from 1.1.0 to 2.0.0
+- Bump github.com/wtsi-npg/extendo from 1.1.0 to 2.0.0
 - Bump github.com/onsi/ginkgo from 1.10.3 to 1.12.0
 - Bump github.com/onsi/gomega from 1.7.1 to 1.9.0
 
@@ -68,7 +95,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Changed
 
-- Bump github.com/kjsanger/extendo from 1.0.0 to 1.1.0
+- Bump github.com/wtsi-npg/extendo from 1.0.0 to 1.1.0
 - Bump github.com/onsi/gomega from 1.5.0 to 1.7.1
 - Bump github.com/onsi/ginkgo from 1.8.0 to 1.10.3
 - Bump github.com/rs/zerolog from 1.14.3 to 1.17.2

Makefile

@@ -1,5 +1,5 @@
 VERSION := $(shell git describe --always --tags --dirty)
-ldflags := "-X github.com/kjsanger/valet/valet.Version=${VERSION}"
+ldflags := "-X github.com/wtsi-npg/valet/valet.Version=${VERSION}"
 build_path = "build/valet-${VERSION}"
 
 .PHONY: build coverage dist install lint test check clean
@@ -11,7 +11,7 @@ install:
 
 build:
 	mkdir -p ${build_path}
-	go build -v -ldflags ${ldflags} -o ${build_path}/valet github.com/kjsanger/valet
+	go build -v -ldflags ${ldflags} -o ${build_path}/valet github.com/wtsi-npg/valet
 
 lint:
 	golangci-lint run ./...

README.md

@@ -3,8 +3,8 @@
 ## Overview
 
 `valet` is a utility for performing data management tasks automatically. Once
-started, `valet` will continue working until interrupted by SIGINT (^C) or
-SIGTERM (kill), when it will stop gracefully.
+started, `valet` will continue working until interrupted by `SIGINT` (^C) or
+`SIGTERM` (kill), when it will stop gracefully.
 
 It monitors filesystem events and also performs filesystem walks to ensure that
 tasks are done in a timely fashion.
@@ -20,24 +20,28 @@ disks filling.
 
 #### Compressing files
 
-Some versions of MinKNOW to do not compress fastq files and no version
-compresses the large sequencing_summary.txt files. `valet` compresses both of
-these types with a simple gzip.
+Some versions of MinKNOW do not compress fastq files and no version
+compresses the large sequencing_summary.txt files. `valet` compresses
+some files with gzip.
 
 - File patterns supported
 
-  - *.fastq$
-  - *.txt$
+  - `*.csv`
+  - `*.fastq`
+  - `*.txt`
 
 #### Archiving files
 
 - Files patterns supported
 
-  - *.csv$
-  - *.fast5$
-  - *.gz$
-  - *.md$
-  - *.pdf$
+  - `*.csv`
+  - `*.fast5`
+  - `*.fastq`
+  - `*.gz`
+  - `*.md`
+  - `*.pdf`
+  - `*.tsv`
+  - `*.txt`
 
 #### Creating up-to-date checksum files
 
@@ -55,7 +59,7 @@ it recognises for archiving.
 
   - Checksum file patterns supported
 
-    - (data file name).md5
+    - `<data file name>.md5`
 
 `valet` will monitor a directory hierarchy and locate data files within it that
 have no accompanying checksum file, or have a checksum file that is stale.
@@ -64,7 +68,7 @@ have no accompanying checksum file, or have a checksum file that is stale.
 ### Operation
 
 `valet` is a command-line program with online help. Once launched it will
-continue to run until signalled with SIGINT (^C) or SIGTERM (kill), when it
+continue to run until signalled with `SIGINT` (^C) or `SIGTERM` (kill), when it
 will stop by cancelling the filesystem monitor and waiting for any running jobs
 to exit.
 
@@ -116,16 +120,16 @@ function, or another) from operating on a particular file concurrently.
 
 ### Bugs
 
-- See [GitHub issues](https://github.com/kjsanger/valet/issues?q=is%3Aissue+is%3Aopen+label%3Abug)
+- See [GitHub issues](https://github.com/wtsi-npg/valet/issues?q=is%3Aissue+is%3Aopen+label%3Abug)
 
 
 #### Status
 
-[![Build Status](https://travis-ci.org/kjsanger/valet.svg?branch=devel)](https://travis-ci.org/kjsanger/valet)
+[![Unit tests](https://github.com/wtsi-npg/valet/actions/workflows/run-tests.yml/badge.svg)](https://github.com/wtsi-npg/valet/actions/workflows/run-tests.yml)
 
 #### Dependencies:
 
-https://github.com/kjsanger/extendo
+https://github.com/wtsi-npg/extendo
 
   Versions >= 2.0.0
 

cmd/archive.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019. Genome Research Ltd. All rights reserved.
+ * Copyright (C) 2019, 2021. Genome Research Ltd. All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,8 +23,8 @@ package cmd
 import (
 	"os"
 
-	logs "github.com/kjsanger/logshim"
 	"github.com/spf13/cobra"
+	logs "github.com/wtsi-npg/logshim"
 )
 
 var archiveCmd = &cobra.Command{

cmd/archive_annotate.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020. Genome Research Ltd. All rights reserved.
+ * Copyright (C) 2020, 2021. Genome Research Ltd. All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,13 +23,13 @@ package cmd
 import (
 	"os"
 
-	ex "github.com/kjsanger/extendo/v2"
-	logs "github.com/kjsanger/logshim"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	ex "github.com/wtsi-npg/extendo/v2"
+	logs "github.com/wtsi-npg/logshim"
 
-	"github.com/kjsanger/valet/utilities"
-	"github.com/kjsanger/valet/valet"
+	"github.com/wtsi-npg/valet/utilities"
+	"github.com/wtsi-npg/valet/valet"
 )
 
 var archAnnotateFlags = &dataFileCliFlags{}

cmd/archive_create.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019, 2020. Genome Research Ltd. All rights reserved.
+ * Copyright (C) 2019, 2020, 2021. Genome Research Ltd. All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -25,12 +25,12 @@ import (
 	"os"
 	"time"
 
-	ex "github.com/kjsanger/extendo/v2"
-	logs "github.com/kjsanger/logshim"
 	"github.com/spf13/cobra"
+	ex "github.com/wtsi-npg/extendo/v2"
+	logs "github.com/wtsi-npg/logshim"
 
-	"github.com/kjsanger/valet/utilities"
-	"github.com/kjsanger/valet/valet"
+	"github.com/wtsi-npg/valet/utilities"
+	"github.com/wtsi-npg/valet/valet"
 )
 
 

cmd/checksum.go

@@ -23,8 +23,8 @@ package cmd
 import (
 	"os"
 
-	logs "github.com/kjsanger/logshim"
 	"github.com/spf13/cobra"
+	logs "github.com/wtsi-npg/logshim"
 )
 
 var checksumFlags = &dataDirCliFlags{}