integration: split tests into nonmem and postrun subdirectories

A subset of the integration tests don't require NONMEM.  It's
convenient to have them in a separate package from the ones that do so
that they can be more easily executed on machines that don't have
NONMEM.

integration/execute.go

@@ -1,4 +1,4 @@
-package bbitest
+package integration
 
 import (
 	"context"
@@ -10,7 +10,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func executeCommand(ctx context.Context, command string, args ...string) (string, error) {
+func ExecuteCommand(ctx context.Context, command string, args ...string) (string, error) {
 	//Find it in path
 	binary, _ := exec.LookPath(command)
 	cmd := exec.CommandContext(ctx, binary, args...)
@@ -40,7 +40,7 @@ func executeCommand(ctx context.Context, command string, args ...string) (string
 }
 
 // nolint:unparam
-func executeCommandNoErrorCheck(ctx context.Context, command string, args ...string) (string, error) {
+func ExecuteCommandNoErrorCheck(ctx context.Context, command string, args ...string) (string, error) {
 	binary, _ := exec.LookPath(command)
 	cmd := exec.CommandContext(ctx, binary, args...)
 	cmd.Env = os.Environ()

integration/nonmem/README.md

@@ -0,0 +1 @@
+This directory contains integration tests that rely on having NONMEM installed.

integration/bbi_expansion_test.go → integration/nonmem/bbi_expansion_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"
@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 
 	"github.com/metrumresearchgroup/wrapt"
@@ -38,7 +39,7 @@ func TestBBIExpandsWithoutPrefix(tt *testing.T) {
 			filepath.Join(scenario.Workpath, "model", targets),
 		}
 
-		output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+		output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 		t.R.NoError(err)
 		t.R.NotEmpty(output)
@@ -107,7 +108,7 @@ func TestBBIExpandsWithPrefix(tt *testing.T) {
 			filepath.Join(scenario.Workpath, "model", targets),
 		}
 
-		output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+		output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 		t.R.NoError(err)
 		t.R.NotEmpty(output)
@@ -177,7 +178,7 @@ func TestBBIExpandsWithPrefixToPartialMatch(tt *testing.T) {
 				filepath.Join(scenario.Workpath, "model", targets),
 			}
 
-			output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+			output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 			t.R.NoError(err)
 			t.R.NotEmpty(output)

integration/bbi_local_test.go → integration/nonmem/bbi_local_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"
@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 
 	"github.com/metrumresearchgroup/wrapt"
@@ -135,7 +136,7 @@ func TestNMFEOptionsEndInScript(tt *testing.T) {
 
 					// Now let's run the script that was generated
 					t.R.NoError(os.Chdir(filepath.Join(scenario.Workpath, m.identifier)))
-					_, err = executeCommand(ctx, filepath.Join(scenario.Workpath, m.identifier, m.identifier+".sh"))
+					_, err = ExecuteCommand(ctx, filepath.Join(scenario.Workpath, m.identifier, m.identifier+".sh"))
 					t.R.NoError(os.Chdir(whereami))
 					t.R.NoError(err)
 

integration/bbi_sge_test.go → integration/nonmem/bbi_sge_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"

integration/config_test.go → integration/nonmem/config_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"

integration/data_test.go → integration/nonmem/data_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"

integration/files.go → integration/nonmem/files.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"io/ioutil"

integration/init_test.go → integration/nonmem/init_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"
@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 
 	"github.com/metrumresearchgroup/wrapt"
@@ -34,7 +35,7 @@ func TestInitialization(tt *testing.T) {
 			scenario.Prepare(t, context.Background())
 
 			t.Run(fmt.Sprintf("init_%s", scenario.identifier), func(t *wrapt.T) {
-				_, err := executeCommand(context.Background(), "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
+				_, err := ExecuteCommand(context.Background(), "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
 				t.R.NoError(err)
 
 				t.A.FileExists(filepath.Join(scenario.Workpath, "bbi.yaml"))

integration/nmqual_test.go → integration/nonmem/nmqual_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"context"

integration/nonmem.go → integration/nonmem/nonmem.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"crypto/md5"

integration/postexecution_test.go → integration/nonmem/postexecution_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"bufio"

integration/setup.go → integration/nonmem/setup.go

@@ -1,4 +1,4 @@
-package bbitest
+package nonmem
 
 import (
 	"archive/tar"
@@ -14,6 +14,8 @@ import (
 	"strconv"
 	"strings"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
+
 	"github.com/metrumresearchgroup/wrapt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
@@ -23,13 +25,6 @@ import (
 var ROOT_EXECUTION_DIR string
 var EXECUTION_DIR string
 
-// constants and variables used in summary tests.
-const SUMMARY_TEST_DIR = "testdata/bbi_summary"
-const SUMMARY_GOLD_DIR = "aa_golden_files"
-const noSuchFileError = "no such file or directory"
-const noFilePresentError = "no file present at"
-const wrongExtensionError = "Must provide path to .lst"
-
 type Scenario struct {
 	Details    ScenarioDetails
 	ctx        context.Context
@@ -67,7 +62,7 @@ func (m Model) Execute(scenario *Scenario, args ...string) (string, error) {
 		filepath.Join(scenario.Workpath, m.filename),
 	}...)
 
-	return executeCommand(scenario.ctx, "bbi", cmdArguments...)
+	return ExecuteCommand(scenario.ctx, "bbi", cmdArguments...)
 }
 
 var ErrNoModelsLocated = errors.New("no model directories were located in the provided scenario")
@@ -437,7 +432,7 @@ func findModelFiles(path string) []string {
 func (scenario *Scenario) Prepare(t *wrapt.T, ctx context.Context) {
 	t.Helper()
 
-	_, err := executeCommand(ctx, "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
+	_, err := ExecuteCommand(ctx, "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
 	t.R.NoError(err)
 
 	fs := afero.NewOsFs()
@@ -457,7 +452,7 @@ func (scenario *Scenario) Prepare(t *wrapt.T, ctx context.Context) {
 
 	t.R.NoError(os.Chdir(scenario.Workpath))
 
-	_, err = executeCommand(ctx, "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
+	_, err = ExecuteCommand(ctx, "bbi", "init", "--dir", os.Getenv("NONMEMROOT"))
 	t.R.NoError(err)
 
 	t.R.NoError(os.Chdir(whereami))

integration/README.md → integration/postrun/README.md

@@ -1,5 +1,5 @@
-# bbitest
-This directory contains integration tests data. Many of these tests must be run in an environment containing a working NONMEM installation and/or worker nodes that can be used to execute jobs. See [`.drone.yml`](/.drone.yml) for examples.
+This directory contains integration tests that do _not_ rely on having
+NONMEM installed.
 
 ## Refreshing golden files
 The `bbi nonmem summary` tests (in `integration/bbi_summary_test.go`) use golden files stored in `integration/testdata/bbi_summary/aa_golden_files/`. These need to be refreshed when the relevant functionality in `bbi` changes. This can be done by running the tests with the `UPDATE_SUMMARY=true` environment variable, like so:

integration/bbi_covcor_test.go → integration/postrun/bbi_covcor_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package postrun
 
 import (
 	"context"
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 
 	"github.com/metrumresearchgroup/wrapt"
@@ -32,7 +33,7 @@ func TestCovCorHappyPath(tt *testing.T) {
 				filepath.Join(SUMMARY_TEST_DIR, mod.name, mod.name),
 			}
 
-			output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+			output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 			t.R.NoError(err)
 			t.R.NotEmpty(output)
@@ -74,7 +75,7 @@ func TestCovCorErrors(tt *testing.T) {
 
 			// try without flag and get error
 			var output string
-			output, err := executeCommandNoErrorCheck(context.Background(), "bbi", commandAndArgs...)
+			output, err := ExecuteCommandNoErrorCheck(context.Background(), "bbi", commandAndArgs...)
 			t.R.Error(err)
 
 			errorMatch := rgx.MatchString(output)

integration/bbi_params_test.go → integration/postrun/bbi_params_test.go

@@ -1,11 +1,12 @@
-package bbitest
+package postrun
 
 import (
 	"context"
 	"os"
 	"path/filepath"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 	"github.com/metrumresearchgroup/wrapt"
 )
@@ -38,7 +39,7 @@ func TestParamsSingleModel(tt *testing.T) {
 				commandAndArgs = append(commandAndArgs, tc.bbiOption)
 			}
 
-			output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+			output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 			t.R.NoError(err)
 			t.R.NotEmpty(output)
@@ -74,7 +75,7 @@ func TestParamsDir(tt *testing.T) {
 				commandAndArgs = append(commandAndArgs, tc.bbiOption)
 			}
 
-			output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+			output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 			t.R.NoError(err)
 			t.R.NotEmpty(output)

integration/bbi_reclean_test.go → integration/postrun/bbi_reclean_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package postrun
 
 import (
 	"context"
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/wrapt"
 )
 
@@ -22,7 +23,7 @@ func TestBBIRecleanBasic(tt *testing.T) {
 	_ = ioutil.WriteFile(fdataCSV, []byte("fake"), 0644)
 	t.A.FileExists(fdataCSV)
 
-	output, err := executeCommand(context.Background(),
+	output, err := ExecuteCommand(context.Background(),
 		"bbi", "nonmem", "reclean", "--recleanLvl=1", "-v", dir)
 	t.R.NoError(err)
 	t.R.NotEmpty(output)
@@ -33,7 +34,7 @@ func TestBBIRecleanBasic(tt *testing.T) {
 func TestBBIRecleanError(tt *testing.T) {
 	t := wrapt.WrapT(tt)
 
-	output, err := executeCommandNoErrorCheck(context.Background(),
+	output, err := ExecuteCommandNoErrorCheck(context.Background(),
 		"bbi", "nonmem", "reclean")
 
 	t.R.NotNil(err)

integration/bbi_summary_test.go → integration/postrun/bbi_summary_test.go

@@ -1,4 +1,4 @@
-package bbitest
+package postrun
 
 import (
 	"context"
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"testing"
 
+	. "github.com/metrumresearchgroup/bbi/integration"
 	"github.com/metrumresearchgroup/bbi/utils"
 
 	"github.com/metrumresearchgroup/wrapt"
@@ -64,7 +65,7 @@ func TestSummaryHappyPath(tt *testing.T) {
 						commandAndArgs = append(commandAndArgs, tc.bbiOption)
 					}
 
-					output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+					output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 					t.R.NoError(err)
 					t.R.NotEmpty(output)
@@ -130,14 +131,14 @@ func TestSummaryArgs(tt *testing.T) {
 					}
 
 					// try without flag and get error
-					output, err := executeCommandNoErrorCheck(context.Background(), "bbi", commandAndArgs...)
+					output, err := ExecuteCommandNoErrorCheck(context.Background(), "bbi", commandAndArgs...)
 					t.R.NotNil(err)
 					errorMatch, _ := regexp.MatchString(tm.errorRegEx, output)
 					t.R.True(errorMatch)
 
 					// append flag and get success
 					commandAndArgs = append(commandAndArgs, tm.bbiArg)
-					output, err = executeCommand(context.Background(), "bbi", commandAndArgs...)
+					output, err = ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 					t.R.NoError(err)
 					t.R.NotEmpty(output)
@@ -216,7 +217,7 @@ func TestSummaryErrors(tt *testing.T) {
 				args = append(args, tc.bbiArgs...)
 			}
 			args = append(args, filepath.Join(SUMMARY_TEST_DIR, tc.testPath))
-			output, err := executeCommandNoErrorCheck(context.Background(), "bbi", args...)
+			output, err := ExecuteCommandNoErrorCheck(context.Background(), "bbi", args...)
 			t.R.NotNil(err)
 			errorMatch, _ := regexp.MatchString(tc.errorMsg, output)
 			t.R.True(errorMatch)
@@ -237,7 +238,7 @@ func TestSummaryHappyPathNoExtension(tt *testing.T) {
 			filepath.Join(SUMMARY_TEST_DIR, mod, mod), // adding no extension should work
 		}
 
-		output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+		output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 		t.R.NoError(err)
 		t.R.NotEmpty(output)
@@ -268,7 +269,7 @@ func TestSummaryHappyPathMultipleModels(tt *testing.T) {
 				if tc.bbiOption != "" {
 					commandAndArgs = append(commandAndArgs, tc.bbiOption)
 				}
-				output, err := executeCommand(context.Background(), "bbi", commandAndArgs...)
+				output, err := ExecuteCommand(context.Background(), "bbi", commandAndArgs...)
 
 				gtd := GoldenFileTestingDetails{
 					outputString: output,
@@ -306,7 +307,7 @@ func TestSummaryPathMultipleModelsError(tt *testing.T) {
 				if tc.bbiOption != "" {
 					commandAndArgs = append(commandAndArgs, tc.bbiOption)
 				}
-				output, err := executeCommandNoErrorCheck(context.Background(),
+				output, err := ExecuteCommandNoErrorCheck(context.Background(),
 					"bbi", commandAndArgs...)
 
 				// Avoid testing plain output with golden file due to

integration/golden_util.go → integration/postrun/golden_util.go

@@ -1,4 +1,4 @@
-package bbitest
+package postrun
 
 import (
 	"fmt"