diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 96ae85ba..dd9bb7fd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -27,7 +27,7 @@ jobs: go-version: ${{ matrix.config.go }} - name: Unit tests shell: bash - run: go list ./... | grep -v integration | xargs go test + run: ./scripts/run-unit-tests - name: Build bbi shell: bash run: | diff --git a/.golangci.yml b/.golangci.yml index d36ad7d8..ea2343ab 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -9,6 +9,7 @@ issues: - gosec exclude-dirs: - (^|/)mock($|/) + - internal/valtools linters-settings: errcheck: diff --git a/Makefile b/Makefile index 54700942..19b58b31 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,7 @@ MAKE_HOME=${PWD} install: cd cmd/bbi; go install ${LDFLAGS} + +VT_TEST_RUNNERS = scripts/run-unit-tests +VT_TEST_RUNNERS += scripts/run-integration-tests +include internal/valtools/rules.mk diff --git a/cmd/bbi/main.go b/cmd/bbi/main.go index 5ec380d2..9a616300 100644 --- a/cmd/bbi/main.go +++ b/cmd/bbi/main.go @@ -7,13 +7,6 @@ import ( // buildTime can be set from LDFLAGS during development. var buildTime string -// if want to generate docs -// -// import "github.com/spf13/cobra/doc" -// err := doc.GenMarkdownTree(cmd.RootCmd, "../../docs/bbi") -// if err != nil { -// panic(err) -// } func main() { cmd.Execute(buildTime) } diff --git a/docs/bbi/bbi.md b/docs/bbi/bbi.md deleted file mode 100644 index bb258b8a..00000000 --- a/docs/bbi/bbi.md +++ /dev/null @@ -1,31 +0,0 @@ -## bbi - -manage and execute models - -### Synopsis - -bbi CLI version 1.0.0 - -### Options - -``` - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - -h, --help help for bbi - --json json tree of output, if possible - --no-cor-file do not use cor file - --no-cov-file do not use cov file - --no-ext-file do not use ext file - --no-grd-file do not use grd file - --no-shk-file do not use shk file - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with locally or nodes to execute on in parallel - -v, --verbose verbose output -``` - -### Subcommands -* [nonmem](nonmem/nonmem.md) - Nonmem model execution -* [version](bbi_version.md) - Check version -* [init](init.md) - Initialize configuration in current directory - - diff --git a/docs/bbi/init.md b/docs/bbi/init.md deleted file mode 100644 index ff5a6c75..00000000 --- a/docs/bbi/init.md +++ /dev/null @@ -1,21 +0,0 @@ -## bbi init - -### Synopsis -Run bbi init to create a bbi.yaml configuration file in the current directory. This will create a `bbi.yaml` file in the current directory containing all the defaults necessary to get started. - -### Options - -``` - -h, --help help for init - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - --json json tree of output, if possible - --no-cor-file do not use cor file - --no-cov-file do not use cov file - --no-ext-file do not use ext file - --no-grd-file do not use grd file - --no-shk-file do not use shk file - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with (default 4) - -v, --verbose verbose output -``` \ No newline at end of file diff --git a/docs/bbi/nonmem/clean/clean.md b/docs/bbi/nonmem/clean/clean.md deleted file mode 100644 index cc802162..00000000 --- a/docs/bbi/nonmem/clean/clean.md +++ /dev/null @@ -1,66 +0,0 @@ -## bbi clean - -clean files and folders - -### Synopsis - - - -glob examples: -bbi clean *.mod // anything with extension .mod -bbi clean *.mod --noFolders // anything with extension .mod -bbi clean run* // anything starting with run -regular expression examples: - -bbi clean ^run --regex // anything beginning with the letters run -bbi clean ^run -v --regex // print out files and folders that will be deleted -bbi clean ^run --filesOnly --regex // only remove matching files -bbi clean _est_ --dirsOnly --regex // only remove matching folders -bbi clean _est_ --dirsOnly --preview --regex // show what output would be if clean occured but don't actually clean -bbi clean "run009.[^mod]" --regex // all matching run009. but not .mod files -bbi clean "run009.(mod|lst)$" --regex // match run009.lst and run009.mod - -can also clean via the opposite of a match with inverse - -bbi clean ".modt{0,1}$" --filesOnly --inverse --regex // clean all files not matching .mod or .modt - -clean copied files via - -bbi clean --copiedRuns="run001" -bbi clean --copiedRuns="run[001:010]" - -can be a comma separated list as well - -bbi clean --copiedRuns="run[001:010],run100" - - -``` -bbi clean [flags] -``` - -### Options - -``` - --copiedRuns string run names - --dirsOnly only match and clean directories - --filesOnly only match and clean files - -h, --help help for clean - --inverse inverse selection from the given regex match criteria - --regex use regular expression to match instead of glob -``` - -### Options inherited from parent commands - -``` - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with - -t, --tree json tree of output, if possible - -v, --verbose verbose output -``` - -### SEE ALSO -* [bbi](bbi.md) - manage and execute models - -###### Auto generated by spf13/cobra on 30-Oct-2017 diff --git a/docs/bbi/nonmem/nonmem.md b/docs/bbi/nonmem/nonmem.md deleted file mode 100644 index e62fd3e7..00000000 --- a/docs/bbi/nonmem/nonmem.md +++ /dev/null @@ -1,91 +0,0 @@ -## bbi nonmem - -Run (and other actions) against nonmem model(s) - -### Synopsis - -Nonmem and its subcommands are all based around the execution and interpretation of nonmem models and more. - -### Options - -``` - -h, --help help for nonmem - --nm_version string Version of nonmem from the configuration list to use - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - --json json tree of output, if possible - --no-cor-file do not use cor file - --no-cov-file do not use cov file - --no-ext-file do not use ext file - --no-grd-file do not use grd file - --no-shk-file do not use shk file - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with (default 4) - -v, --verbose verbose output -``` - -### Subcommands -* [clean](clean/clean.md) -* [probs](probs/probs.md) -* [reclean](reclean/reclean.md) -* [run](run/run.md) -* [scaffold](scaffold/scaffold.md) -* [summary](summary/summary.md) - - -### nmVersion -The nm_version flag is a string representing a key in the bbi.yml loaded from the model directory. Given the below sample: - -```yml -nm_version: nm74gf -overwrite: false -clean_lvl: 1 -copy_lvl: 0 -git: true -save_config: false -output_dir: '{{ .Name }}' -threads: 4 -debug: false -local: - create_child_dirs: false -nonmem: - nm73gf: - home: /opt/NONMEM/nm73gf - executable: nmfe73 - nmqual: true - default: false - nm73gf_nmfe: - home: /opt/NONMEM/nm73gf_nmfe - executable: nmfe73 - nmqual: false - default: false - nm74gf: - home: /opt/NONMEM/nm74gf - executable: nmfe74 - nmqual: true - default: false - nm74gf_nmfe: - home: /opt/NONMEM/nm74gf_nmfe - executable: nmfe74 - nmqual: false - default: false -parallel: false -delay: 0 -nmqual: false -json: false -log_file: "" -nmfe_options: - license_file: "" - prsame: false - background: false - prcompile: false - prdefault: false - tprdefault: false - nobuild: false - maxlim: 100 -mpi_exec_path: /usr/local/mpich3/bin/mpiexec -parallel_timeout: 2147483647 -parafile: "" -``` - -Using an `nmVersion` of nm74_gf would load the nonmem binary named `nmfe74` from `ls/opt/NONMEM/nm74gf/run` during nonmem local execution diff --git a/docs/bbi/nonmem/probs/probs.md b/docs/bbi/nonmem/probs/probs.md deleted file mode 100644 index ff7cb800..00000000 --- a/docs/bbi/nonmem/probs/probs.md +++ /dev/null @@ -1,36 +0,0 @@ -## bbi probs - -summarize information about project - -### Synopsis - - -get information about models in the project: -nmu project - - -``` -bbi probs [flags] -``` - -### Options - -``` - -h, --help help for probs -``` - -### Options inherited from parent commands - -``` - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with - -t, --tree json tree of output, if possible - -v, --verbose verbose output -``` - -### SEE ALSO -* [bbi](bbi.md) - manage and execute models - -###### Auto generated by spf13/cobra on 30-Oct-2017 diff --git a/docs/bbi/nonmem/reclean/reclean.md b/docs/bbi/nonmem/reclean/reclean.md deleted file mode 100644 index 1863e63c..00000000 --- a/docs/bbi/nonmem/reclean/reclean.md +++ /dev/null @@ -1,37 +0,0 @@ -## bbi reclean - -clean files in an estimation directory by clean level - -### Synopsis - - - - bbi reclean run001_est_01 - - -``` -bbi reclean [flags] -``` - -### Options - -``` - --cleanLvl int clean level to apply (default 5) - -h, --help help for reclean -``` - -### Options inherited from parent commands - -``` - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with - -t, --tree json tree of output, if possible - -v, --verbose verbose output -``` - -### SEE ALSO -* [bbi](bbi.md) - manage and execute models - -###### Auto generated by spf13/cobra on 30-Oct-2017 diff --git a/docs/bbi/nonmem/run/local/local.md b/docs/bbi/nonmem/run/local/local.md deleted file mode 100644 index 1dc8eba8..00000000 --- a/docs/bbi/nonmem/run/local/local.md +++ /dev/null @@ -1,48 +0,0 @@ -## bbi nonmem run local - -Targets a model or collection of models for execution on the local machine - -### Synopsis - -Specifies to run the targeted model on the local machine - -### Options - -``` ---create_child_dirs Indicates whether or not local branch executionshould create a new subdirectory with the output_dir variable as its name and execute in that directory (defaulttrue) -``` - -The `create_child_dirs` flag is used to determine whether a new directory should be created for nonmem to place its -output into. By default this is no, but whenever a job is run in the SGE mode, the SGE phase will create a directory -so the local mode of execution will be set to no such that it does not create another child directory underneath. - -This also avoids issues dealing with overwrite. This pattern allows us to maintain resiliency of files across execution -modes and not have to have constant override logic for `overwrite` - -### Sample Output -``` -$ ./bbi nonmem run local 240/[001:009].mod -2019/12/23 18:12:46 expanding model pattern: 240/[001:009].mod -2019/12/23 18:12:46 Configuration file successfully loaded from 240/bbi.yaml -2019/12/23 18:12:46 A total of 9 models have been located for work -2019/12/23 18:12:46 9 models successfully completed initial setup phase. -2019/12/23 18:12:46 Beginning local work phase for 001 -2019/12/23 18:12:46 Beginning local work phase for 002 -2019/12/23 18:12:53 Beginning cleanup phase for model 002 -2019/12/23 18:12:53 Beginning local work phase for 003 -2019/12/23 18:12:53 Beginning cleanup phase for model 001 -2019/12/23 18:12:53 Beginning local work phase for 004 -2019/12/23 18:13:01 Beginning cleanup phase for model 003 -2019/12/23 18:13:01 Beginning local work phase for 005 -2019/12/23 18:13:02 Beginning cleanup phase for model 004 -2019/12/23 18:13:02 Beginning local work phase for 006 -2019/12/23 18:13:09 Beginning cleanup phase for model 005 -2019/12/23 18:13:09 Beginning local work phase for 007 -2019/12/23 18:13:09 Beginning cleanup phase for model 006 -2019/12/23 18:13:09 Beginning local work phase for 008 -2019/12/23 18:13:17 Beginning cleanup phase for model 008 -2019/12/23 18:13:17 Beginning local work phase for 009 -2019/12/23 18:13:17 Beginning cleanup phase for model 007 -2019/12/23 18:13:23 Beginning cleanup phase for model 009 -9 models completed in 37.964110535s -``` \ No newline at end of file diff --git a/docs/bbi/nonmem/run/run.md b/docs/bbi/nonmem/run/run.md deleted file mode 100644 index e3e52657..00000000 --- a/docs/bbi/nonmem/run/run.md +++ /dev/null @@ -1,150 +0,0 @@ -## bbi nonmem run - -Targets a model or collection of models for execution - -### Synopsis - -The run command collects a series of flags necessary to define the behavior for: - -* How to run a nonmem model - * Locally - * Sun Grid Engine -* How (if at all) to sanitize the output directory -* How (if at all) to copy result files -* How (if at all) to handle gitignore files - -#### Important Flags and Definitions - -* `--overwrite=true` : Specifies that, if an output directory for a given model exists, to remove all of it's contents and re-run the model. Default is true -* `--git=true` : Specifies the following: - * During initial model execution, a wildcard (`*`) gitignore file is placed into the model execution directory - * While not explicitly necessary, if you are using something like RStudio or other platform that is tracking changes to git, this will keep those other platforms sane while nonmem is generating all of its temporary files. Vastly important when multiple runs are being done at once. - * After execution is done, the gitignore file is updated with various temp files to prevent them from being committed into a repo on accident -* `--output_dir {{ .Name }}` : A valid go-template (Defaults to the one listed here) that will be used for creating the directories in which model execution will occur. `Name` is the only variable interpreted, but you can append / prepend items to it: - * `'{{ .Name }}_output` : yields modelname_output - * `'output_{{ .Name }}'` : yields output_modelname -* `--clean_lvl <1|2|3>` : Based on a list of extensions and files (See below), will remove any matching files from the output directory after the work is done. Default is 2 -* `--copy_lvl <1|2|3>` : Based on a list of extension and files (See below), will remove copy any of the matched files back into the original model directory prepended with the model name. Mirrors PSN functionality, although the default is 0 (or off) - -### Options - -``` - --clean_lvl int clean level used for file output from a given (set of) runs (default 1) - --config string Path (relative or absolute) to another bbi.yaml to load - --copy_lvl int copy level used for file output from a given (set of) runs - --delay int Selects a random number of seconds between 1 and this value to stagger / jitter job execution. Assists in dealing with large volumes of work dealing with the same data set. May avoid NMTRAN issues about not being able read / close files - --git whether git is used - -h, --help help for run - --log_file string If populated, specifies the file into which to store the output / logging details from bbi - --output_dir string Go template for the output directory to use for storing details of each executed model (default "{{ .Name }}") - --overwrite Whether or not to remove existing output directories if they are present - --save_config Whether or not to save the existing configuration to a file with the model (default true) -``` - -### Subcommands -* [local](local/local.md) - Nonmem model execution -* [sge](sge/sge.md) - check version - - -### Turnstile Execution Control -The run command and its variants all implement the [turnstile](https://github.com/metrumresearchgroup/turnstile) workflow to manage concurrency of model execution. At the top level, bbi takes a `--threads` option. Whatever this value is set to is the maximum amount of ongoing work turnstile will allow. For local and grid execution this allows you to controllably stagger the work being doled out. - -### Clean Level Files -Currently there is really only one level of clean files, which is level 1. Its contents are the temporary files (including the nonmem executable) created during nonmem execution: - -``` - "background.set", - "compile.lnk", - "FCON", - "FDATA", - "FMSG", - "FREPORT", - "FSIZES", - "FSTREAM", - "FSUBS", - "FSUBS.0", - "FSUBS.o", - "FSUBS_MU.F90", - "FSUBS.f90", - "fsubs.f90", - "FSUBS2", - "gfortran.txt", - "GFCOMPILE.BAT", - "INTER", - "licfile.set", - "linkc.lnk", - "LINK.LNK", - "LINKC.LNK", - "locfile.set", - "maxlim.set", - "newline", - "nmexec.set", - "nmpathlist.txt", - "nmprd4p.mod", - "nobuild.set", - "parafile.set", - "parafprint.set", - "prcompile.set", - "prdefault.set", - "prsame.set", - "PRSIZES.f90", - "rundir.set", - "runpdir.set", - "simparon.set", - "temp_dir", - "tprdefault.set", - "trskip.set", - "worker.set", - "xmloff.set", - "fort.2001", - "fort.2002", - "flushtime.set", - "nonmem" -``` - -### Copy Level Files - -Copy-up funtionality is broken out into three tiers - -#### Tier 1 -Tier 1 consists of: - -* Any files named in the nonmem model table definitions -* Any files beginning with the model filename and ending with the following extensions: - -``` - ".xml", - ".grd", - ".shk", - ".cor", - ".cov", - ".ext", - ".lst" -``` - -#### Tier 2 -Any files beginning with the model filename and ending with the following extensions: - -``` - ".clt", - ".coi", - ".clt", - ".coi", - ".cpu", - ".shm", - ".phi", -``` - - -#### Tier 3 -Any files beginning with the model filename and ending with the followin extensions: - -``` - "_ETAS", - "_RMAT", - "_SMAT", - ".msf", - "_ETAS.msf", - "_RMAT.msf", - "_SMAT.msf", -``` \ No newline at end of file diff --git a/docs/bbi/nonmem/run/sge/sge.md b/docs/bbi/nonmem/run/sge/sge.md deleted file mode 100644 index 1bbcab42..00000000 --- a/docs/bbi/nonmem/run/sge/sge.md +++ /dev/null @@ -1,25 +0,0 @@ -## bbi nonmem run sge - -Targets a model or collection of models for execution on the Sun Grid Engine - -### Synopsis - -Specifies to run the targeted model on the Sun Grid Engine. The SGE execution method basically operates as a qsub wrapper for `bbi nonmem run local` to ensure uniformity. - -### Sample Output -``` -2019/12/23 18:22:07 expanding model pattern:[001:009].mod -2019/12/23 18:22:07 Configuration file successfully loaded from /240/bbi.yaml -2019/12/23 18:22:07 A total of 9 models have been located for work -2019/12/23 18:22:07 9 models successfully completed initial setup phase. -2019/12/23 18:22:07 Beginning SGE work phase for 001 -2019/12/23 18:22:07 Beginning SGE work phase for 002 -2019/12/23 18:22:07 Beginning SGE work phase for 003 -2019/12/23 18:22:07 Beginning SGE work phase for 004 -2019/12/23 18:22:07 Beginning SGE work phase for 005 -2019/12/23 18:22:07 Beginning SGE work phase for 006 -2019/12/23 18:22:07 Beginning SGE work phase for 007 -2019/12/23 18:22:07 Beginning SGE work phase for 008 -2019/12/23 18:22:07 Beginning SGE work phase for 009 -9 models completed in 22.986631ms -``` diff --git a/docs/bbi/nonmem/scaffold/scaffold.md b/docs/bbi/nonmem/scaffold/scaffold.md deleted file mode 100644 index 1447fd41..00000000 --- a/docs/bbi/nonmem/scaffold/scaffold.md +++ /dev/null @@ -1,39 +0,0 @@ -## bbi scaffold - -scaffold directory structures - -### Synopsis - - - - nmu scaffold --cacheDir=nmcache - - nmu scaffold --cacheDir=../nmcache --preview // show where the cache dir would be created - - -``` -bbi scaffold [flags] -``` - -### Options - -``` - --cacheDir string create cache directory at path/name - -h, --help help for scaffold -``` - -### Options inherited from parent commands - -``` - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with - -t, --tree json tree of output, if possible - -v, --verbose verbose output -``` - -### SEE ALSO -* [bbi](bbi.md) - manage and execute models - -###### Auto generated by spf13/cobra on 30-Oct-2017 diff --git a/docs/bbi/nonmem/summary/summary.md b/docs/bbi/nonmem/summary/summary.md deleted file mode 100644 index ddbe6eae..00000000 --- a/docs/bbi/nonmem/summary/summary.md +++ /dev/null @@ -1,46 +0,0 @@ -## bbi summary - -summarize the output of a model - -### Synopsis - - -run model(s), for example: -nmu summarize run001.lst - - -``` -bbi nonmem summary [flags] -``` - -### Options - -``` - -h, --help help for summary -``` - -### Options inherited from parent commands - -``` - -h, --help help for nonmem - --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") - --nm_version string Version of nonmem from the configuration list to use - --nodes int The number of nodes on which to perform parallel operations (default 8) - --parallel Whether or not to run nonmem in parallel mode - --timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 30) - --config string config file (default is bbi.yaml is directory where command is run) - -d, --debug debug mode - --json json tree of output, if possible - --no-cor-file do not use cor file - --no-cov-file do not use cov file - --no-ext-file do not use ext file - --no-grd-file do not use grd file - --no-shk-file do not use shk file - -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with (default 4) - -v, --verbose verbose output -``` - - -### SEE ALSO -* [bbi](bbi.md) - manage and execute models diff --git a/docs/commands/bbi.md b/docs/commands/bbi.md new file mode 100644 index 00000000..8eb21687 --- /dev/null +++ b/docs/commands/bbi.md @@ -0,0 +1,22 @@ +## bbi + +manage and execute models + +### Options + +``` + -d, --debug debug mode + -h, --help help for bbi + --json json tree of output, if possible + -o, --output string output file + -p, --preview preview action, but don't actually run command + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi init](bbi_init.md) - Create configuration file with defaults +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid +* [bbi version](bbi_version.md) - check version + diff --git a/docs/commands/bbi_init.md b/docs/commands/bbi_init.md new file mode 100644 index 00000000..f63f6baa --- /dev/null +++ b/docs/commands/bbi_init.md @@ -0,0 +1,35 @@ +## bbi init + +Create configuration file with defaults + +### Synopsis + +Run bbi init to create a bbi.yaml configuration file in the current directory. + + +``` +bbi init [flags] +``` + +### Options + +``` + --dir strings A directory in which to look for NonMem Installations + -h, --help help for init +``` + +### Options inherited from parent commands + +``` + -d, --debug debug mode + --json json tree of output, if possible + -o, --output string output file + -p, --preview preview action, but don't actually run command + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi](bbi.md) - manage and execute models + diff --git a/docs/commands/bbi_nonmem.md b/docs/commands/bbi_nonmem.md new file mode 100644 index 00000000..661e36ef --- /dev/null +++ b/docs/commands/bbi_nonmem.md @@ -0,0 +1,74 @@ +## bbi nonmem + +nonmem a (set of) models locally or on the grid + +### Synopsis + + +run nonmem model(s), for example: +bbi nonmem run run001.mod +bbi nonmem run --clean_lvl=1 run001.mod run002.mod +bbi nonmem run run[001:006].mod // expand to run001.mod run002.mod ... run006.mod local +bbi nonmem run .// run all models in directory + + +summarize model(s), for example: +bbi nonmem summary run001/run001 +bbi nonmem summary run001/run001.lst +bbi nonmem summary run001/run001.res +bbi nonmem summary run001/run001 run002/run002 + + +load .cov and .cor output from model(s), for example: +bbi nonmem covcor run001/run001 +bbi nonmem covcor run001/run001.cov + + + +``` +bbi nonmem [flags] +``` + +### Options + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -h, --help help for nonmem + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault +``` + +### Options inherited from parent commands + +``` + -d, --debug debug mode + --json json tree of output, if possible + -o, --output string output file + -p, --preview preview action, but don't actually run command + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi](bbi.md) - manage and execute models +* [bbi nonmem clean](bbi_nonmem_clean.md) - clean files and folders +* [bbi nonmem covcor](bbi_nonmem_covcor.md) - load .cov and .cor output from a model run +* [bbi nonmem params](bbi_nonmem_params.md) - get the parameters of model(s) +* [bbi nonmem probs](bbi_nonmem_probs.md) - summarize information about project +* [bbi nonmem reclean](bbi_nonmem_reclean.md) - clean files in an estimation directory by clean level +* [bbi nonmem run](bbi_nonmem_run.md) - run a (set of) models locally or on the grid +* [bbi nonmem scaffold](bbi_nonmem_scaffold.md) - scaffold directory structures +* [bbi nonmem summary](bbi_nonmem_summary.md) - summarize the output of model(s) + diff --git a/docs/commands/bbi_nonmem_clean.md b/docs/commands/bbi_nonmem_clean.md new file mode 100644 index 00000000..58107ae3 --- /dev/null +++ b/docs/commands/bbi_nonmem_clean.md @@ -0,0 +1,79 @@ +## bbi nonmem clean + +clean files and folders + +### Synopsis + + +glob examples: +bbi clean *.mod // anything with extension .mod +bbi clean *.mod --noFolders // anything with extension .mod +bbi clean run* // anything starting with run +regular expression examples: + +bbi clean ^run --regex // anything beginning with the letters run +bbi clean ^run -v --regex // print out files and folders that will be deleted +bbi clean ^run --filesOnly --regex // only remove matching files +bbi clean _est_ --dirsOnly --regex // only remove matching folders +bbi clean _est_ --dirsOnly --preview --regex // show what output would be if clean occured but don't actually clean +bbi clean "run009.[^mod]" --regex // all matching run009. but not .mod files +bbi clean "run009.(mod|lst)$" --regex // match run009.lst and run009.mod + +can also clean via the opposite of a match with inverse + +bbi clean ".modt{0,1}$" --filesOnly --inverse --regex // clean all files not matching .mod or .modt + +clean copied files via + +bbi clean --copiedRuns="run001" +bbi clean --copiedRuns="run[001:010]" + +can be a comma separated list as well + +bbi clean --copiedRuns="run[001:010],run100" + + +``` +bbi nonmem clean [flags] +``` + +### Options + +``` + --copiedRuns string run names + --dirsOnly only match and clean directories + --filesOnly only match and clean files + -h, --help help for clean + --inverse inverse selection from the given regex match criteria + --regex use regular expression to match instead of glob +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_covcor.md b/docs/commands/bbi_nonmem_covcor.md new file mode 100644 index 00000000..3ef2d1ef --- /dev/null +++ b/docs/commands/bbi_nonmem_covcor.md @@ -0,0 +1,50 @@ +## bbi nonmem covcor + +load .cov and .cor output from a model run + +### Synopsis + +load .cov and .cor output from model(s), for example: +bbi nonmem covcor run001/run001 +bbi nonmem covcor run001/run001.cov + + +``` +bbi nonmem covcor [flags] +``` + +### Options + +``` + -h, --help help for covcor +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_params.md b/docs/commands/bbi_nonmem_params.md new file mode 100644 index 00000000..ff6bdfcd --- /dev/null +++ b/docs/commands/bbi_nonmem_params.md @@ -0,0 +1,54 @@ +## bbi nonmem params + +get the parameters of model(s) + +### Synopsis + +summarize model(s), for example: +bbi nonmem params run001 +bbi nonmem params run001 +bbi nonmem params run001 + + +``` +bbi nonmem params [flags] +``` + +### Options + +``` + --dir string name of directory to look for runs + --ext-file string name of custom ext-file + -h, --help help for params + --no-names don't print a header of names +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_probs.md b/docs/commands/bbi_nonmem_probs.md new file mode 100644 index 00000000..b4735a32 --- /dev/null +++ b/docs/commands/bbi_nonmem_probs.md @@ -0,0 +1,49 @@ +## bbi nonmem probs + +summarize information about project + +### Synopsis + +get information about models in the project: +nmu project + + +``` +bbi nonmem probs [flags] +``` + +### Options + +``` + -h, --help help for probs +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_reclean.md b/docs/commands/bbi_nonmem_reclean.md new file mode 100644 index 00000000..3069e767 --- /dev/null +++ b/docs/commands/bbi_nonmem_reclean.md @@ -0,0 +1,50 @@ +## bbi nonmem reclean + +clean files in an estimation directory by clean level + +### Synopsis + + + bbi reclean run001_est_01 + + +``` +bbi nonmem reclean [flags] +``` + +### Options + +``` + -h, --help help for reclean + --recleanLvl int clean level to apply +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_run.md b/docs/commands/bbi_nonmem_run.md new file mode 100644 index 00000000..c57cba6e --- /dev/null +++ b/docs/commands/bbi_nonmem_run.md @@ -0,0 +1,65 @@ +## bbi nonmem run + +run a (set of) models locally or on the grid + +### Synopsis + +run nonmem model(s), for example: +bbi nonmem run run001.mod +bbi nonmem run --clean_lvl=1 run001.mod run002.mod +bbi nonmem run run[001:006].mod // expand to run001.mod run002.mod ... run006.mod local +bbi nonmem run .// run all models in directory + + +``` +bbi nonmem run [flags] +``` + +### Options + +``` + --additional_post_work_envs strings Any additional values (as ENV KEY=VALUE) to provide for the post execution environment + --clean_lvl int clean level used for file output from a given (set of) runs (default 1) + --config string Path (relative or absolute) to another bbi.yaml to load + --copy_lvl int copy level used for file output from a given (set of) runs + --delay int Selects a random number of seconds between 1 and this value to stagger / jitter job execution. Assists in dealing with large volumes of work dealing with the same data set. May avoid NMTRAN issues about not being able read / close files + --git whether git is used + -h, --help help for run + --log_file string If populated, specifies the file into which to store the output / logging details from bbi + --output_dir string Go template for the output directory to use for storing details of each executed model (default "{{ .Name }}") + --overwrite Whether or not to remove existing output directories if they are present + --post_work_executable string A script or binary to run when job execution completes or fails + --save_config Whether or not to save the existing configuration to a file with the model (default true) +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid +* [bbi nonmem run local](bbi_nonmem_run_local.md) - local specifies to run a (set of) models locally +* [bbi nonmem run sge](bbi_nonmem_run_sge.md) - sge specifies to run a (set of) models on the Sun Grid Engine + diff --git a/docs/commands/bbi_nonmem_run_local.md b/docs/commands/bbi_nonmem_run_local.md new file mode 100644 index 00000000..8f29a62f --- /dev/null +++ b/docs/commands/bbi_nonmem_run_local.md @@ -0,0 +1,64 @@ +## bbi nonmem run local + +local specifies to run a (set of) models locally + +### Synopsis + +run nonmem model(s), for example: +bbi nonmem run run001.mod +bbi nonmem run --clean_lvl=1 run001.mod run002.mod +bbi nonmem run run[001:006].mod // expand to run001.mod run002.mod ... run006.mod local +bbi nonmem run .// run all models in directory + + +``` +bbi nonmem run local [flags] +``` + +### Options + +``` + --create_child_dirs Indicates whether or not local branch executionshould create a new subdirectory with the output_dir variable as its name and execute in that directory (default true) + -h, --help help for local +``` + +### Options inherited from parent commands + +``` + --additional_post_work_envs strings Any additional values (as ENV KEY=VALUE) to provide for the post execution environment + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + --clean_lvl int clean level used for file output from a given (set of) runs (default 1) + --config string Path (relative or absolute) to another bbi.yaml to load + --copy_lvl int copy level used for file output from a given (set of) runs + -d, --debug debug mode + --delay int Selects a random number of seconds between 1 and this value to stagger / jitter job execution. Assists in dealing with large volumes of work dealing with the same data set. May avoid NMTRAN issues about not being able read / close files + --git whether git is used + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --log_file string If populated, specifies the file into which to store the output / logging details from bbi + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --output_dir string Go template for the output directory to use for storing details of each executed model (default "{{ .Name }}") + --overwrite Whether or not to remove existing output directories if they are present + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --post_work_executable string A script or binary to run when job execution completes or fails + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --save_config Whether or not to save the existing configuration to a file with the model (default true) + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem run](bbi_nonmem_run.md) - run a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_run_sge.md b/docs/commands/bbi_nonmem_run_sge.md new file mode 100644 index 00000000..ebc6a404 --- /dev/null +++ b/docs/commands/bbi_nonmem_run_sge.md @@ -0,0 +1,65 @@ +## bbi nonmem run sge + +sge specifies to run a (set of) models on the Sun Grid Engine + +### Synopsis + +run nonmem model(s), for example: +bbi nonmem run run001.mod +bbi nonmem run --clean_lvl=1 run001.mod run002.mod +bbi nonmem run run[001:006].mod // expand to run001.mod run002.mod ... run006.mod local +bbi nonmem run .// run all models in directory + + +``` +bbi nonmem run sge [flags] +``` + +### Options + +``` + --bbi_binary string path to bbi executable to be called in goroutines (SGE Execution) + --grid_name_prefix string Any prefix you wish to add to the name of jobs being submitted to the grid + -h, --help help for sge +``` + +### Options inherited from parent commands + +``` + --additional_post_work_envs strings Any additional values (as ENV KEY=VALUE) to provide for the post execution environment + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + --clean_lvl int clean level used for file output from a given (set of) runs (default 1) + --config string Path (relative or absolute) to another bbi.yaml to load + --copy_lvl int copy level used for file output from a given (set of) runs + -d, --debug debug mode + --delay int Selects a random number of seconds between 1 and this value to stagger / jitter job execution. Assists in dealing with large volumes of work dealing with the same data set. May avoid NMTRAN issues about not being able read / close files + --git whether git is used + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --log_file string If populated, specifies the file into which to store the output / logging details from bbi + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --output_dir string Go template for the output directory to use for storing details of each executed model (default "{{ .Name }}") + --overwrite Whether or not to remove existing output directories if they are present + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --post_work_executable string A script or binary to run when job execution completes or fails + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --save_config Whether or not to save the existing configuration to a file with the model (default true) + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem run](bbi_nonmem_run.md) - run a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_scaffold.md b/docs/commands/bbi_nonmem_scaffold.md new file mode 100644 index 00000000..dcabb3fa --- /dev/null +++ b/docs/commands/bbi_nonmem_scaffold.md @@ -0,0 +1,52 @@ +## bbi nonmem scaffold + +scaffold directory structures + +### Synopsis + + + nmu scaffold --cacheDir=nmcache + + nmu scaffold --cacheDir=../nmcache --preview // show where the cache dir would be created + + +``` +bbi nonmem scaffold [flags] +``` + +### Options + +``` + --cacheDir string create cache directory at path/name + -h, --help help for scaffold +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/commands/bbi_nonmem_summary.md b/docs/commands/bbi_nonmem_summary.md new file mode 100644 index 00000000..f00cb692 --- /dev/null +++ b/docs/commands/bbi_nonmem_summary.md @@ -0,0 +1,56 @@ +## bbi nonmem summary + +summarize the output of model(s) + +### Synopsis + +summarize model(s), for example: +bbi nonmem summary run001/run001 +bbi nonmem summary run001/run001.lst +bbi nonmem summary run001/run001.res +bbi nonmem summary run001/run001 run002/run002 + + +``` +bbi nonmem summary [flags] +``` + +### Options + +``` + --ext-file string name of custom ext-file + -h, --help help for summary + --no-ext-file do not use ext file + --no-grd-file do not use grd file + --no-shk-file do not use shk file +``` + +### Options inherited from parent commands + +``` + --background RAW NMFE OPTION - Tells nonmem not to scan StdIn for control characters + -d, --debug debug mode + --json json tree of output, if possible + --licfile string RAW NMFE OPTION - Specify a license file to use with NMFE (Nonmem) + --maxlim int RAW NMFE OPTION - Set the maximum values for the buffers used by Nonmem (if 0, don't pass -maxlim to nmfe) (default 2) + --mpi_exec_path string The fully qualified path to mpiexec. Used for nonmem parallel operations (default "/usr/local/mpich3/bin/mpiexec") + --nm_version string Version of nonmem from the configuration list to use + --nmqual Whether or not to execute with nmqual (autolog.pl) + --nobuild RAW NMFE OPTION - Skips recompiling and rebuilding on nonmem executable + -o, --output string output file + --parafile string Location of a user-provided parafile to use for parallel execution + --parallel Whether or not to run nonmem in parallel mode + --parallel_timeout int The amount of time to wait for parallel operations in nonmem before timing out (default 2147483647) + --prcompile RAW NMFE OPTION - Forces PREDPP compilation + --prdefault RAW NMFE OPTION - Do not recompile any routines other than FSUBS + -p, --preview preview action, but don't actually run command + --prsame RAW NMFE OPTION - Indicates to nonmem that the PREDPP compilation step should be skipped + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) + --tprdefault RAW NMFE OPTION - Test if is okay to do -prdefault + -v, --verbose verbose output +``` + +### SEE ALSO + +* [bbi nonmem](bbi_nonmem.md) - nonmem a (set of) models locally or on the grid + diff --git a/docs/bbi/bbi_version.md b/docs/commands/bbi_version.md similarity index 60% rename from docs/bbi/bbi_version.md rename to docs/commands/bbi_version.md index a3da86b4..287df00c 100644 --- a/docs/bbi/bbi_version.md +++ b/docs/commands/bbi_version.md @@ -4,9 +4,8 @@ check version ### Synopsis - check the current bbi version -bbi version +bbi version ``` @@ -22,15 +21,15 @@ bbi version [flags] ### Options inherited from parent commands ``` - --config string config file (default is bbi.yaml is directory where command is run) -d, --debug debug mode + --json json tree of output, if possible + -o, --output string output file -p, --preview preview action, but don't actually run command - --threads int number of threads to execute with - -t, --tree json tree of output, if possible + --threads int number of threads to execute with locally or nodes to execute on in parallel (default 4) -v, --verbose verbose output ``` ### SEE ALSO + * [bbi](bbi.md) - manage and execute models -###### Auto generated by spf13/cobra on 30-Oct-2017 diff --git a/docs/validation/matrix.yaml b/docs/validation/matrix.yaml new file mode 100644 index 00000000..091d1e7e --- /dev/null +++ b/docs/validation/matrix.yaml @@ -0,0 +1,91 @@ +- entrypoint: bbi + skip: true + +- entrypoint: bbi init + code: cmd/init.go + doc: docs/commands/bbi_init.md + tests: + - cmd/init_test.go + - integration/nonmem/init_test.go + +- entrypoint: bbi nonmem + skip: true + +- entrypoint: bbi nonmem clean + code: cmd/clean.go + doc: docs/commands/bbi_nonmem_clean.md + tests: [] + +- entrypoint: bbi nonmem covcor + code: cmd/covcor.go + doc: docs/commands/bbi_nonmem_covcor.md + tests: + - integration/postrun/bbi_covcor_test.go + - parsers/nmparser/read_cov_test.go + +- entrypoint: bbi nonmem params + code: cmd/params.go + doc: docs/commands/bbi_nonmem_params.md + tests: + - cmd/params_test.go + - integration/postrun/bbi_params_test.go + - parsers/nmparser/read_ext_fast_test.go + +- entrypoint: bbi nonmem probs + code: cmd/project.go + doc: docs/commands/bbi_nonmem_probs.md + tests: [] + +- entrypoint: bbi nonmem reclean + code: cmd/reclean.go + doc: docs/commands/bbi_nonmem_reclean.md + tests: + - integration/postrun/bbi_reclean_test.go + +- entrypoint: bbi nonmem run + skip: true + +- entrypoint: bbi nonmem run local + code: cmd/local.go + doc: docs/commands/bbi_nonmem_run_local.md + tests: + - cmd/nonmem_test.go + - integration/nonmem/bbi_local_test.go + - integration/nonmem/config_test.go + - integration/nonmem/data_test.go + - integration/nonmem/nmqual_test.go + - integration/nonmem/postexecution_test.go + - parsers/nmparser/add_path_level_test.go + +- entrypoint: bbi nonmem run sge + code: cmd/sge.go + doc: docs/commands/bbi_nonmem_run_sge.md + tests: + - cmd/nonmem_test.go + - cmd/sge_test.go + - integration/nonmem/bbi_sge_test.go + - parsers/nmparser/add_path_level_test.go + +- entrypoint: bbi nonmem scaffold + code: cmd/scaffold.go + doc: docs/commands/bbi_nonmem_scaffold.md + tests: [] + +- entrypoint: bbi nonmem summary + code: cmd/summary.go + doc: docs/commands/bbi_nonmem_summary.md + tests: + - integration/postrun/bbi_summary_test.go + - parsers/nmparser/parse_block_result_test.go + - parsers/nmparser/parse_final_parameter_estimates_test.go + - parsers/nmparser/parse_lst_file_test.go + - parsers/nmparser/parse_run_details.go + - parsers/nmparser/parse_theta_results_test.go + - parsers/nmparser/read_ext_test.go + - parsers/nmparser/read_grd_test.go + - parsers/nmparser/read_shk_test.go + +- entrypoint: bbi version + code: cmd/version.go + doc: docs/commands/bbi_version.md + tests: [] diff --git a/go.mod b/go.mod index d7054a90..da107ed2 100644 --- a/go.mod +++ b/go.mod @@ -27,8 +27,9 @@ require ( github.com/spf13/cobra v1.1.1 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.7.1 - golang.org/x/tools v0.1.7 // indirect + golang.org/x/mod v0.20.0 + golang.org/x/tools v0.24.0 gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index f3e7a1dd..6a0766f6 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,15 +70,14 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -170,13 +170,11 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= @@ -202,6 +200,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= @@ -209,6 +208,7 @@ github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4Qn github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE= github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -249,7 +249,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -264,6 +264,11 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -283,8 +288,13 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -302,9 +312,16 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -314,7 +331,13 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -338,17 +361,39 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -370,12 +415,15 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -403,7 +451,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -416,7 +463,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/tools/docgen/main.go b/internal/tools/docgen/main.go new file mode 100644 index 00000000..cf825316 --- /dev/null +++ b/internal/tools/docgen/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "os" + + "github.com/metrumresearchgroup/bbi/cmd" + + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" +) + +func gen(cmd *cobra.Command, dir string) error { + if err := os.RemoveAll(dir); err != nil { + return err + } + + if err := os.MkdirAll(dir, 0777); err != nil { + return err + } + + return doc.GenMarkdownTree(cmd, dir) +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "usage: %s \n", os.Args[0]) + os.Exit(2) + } + + outdir := os.Args[1] + + root := cmd.NewRootCmd() + root.Long = "" // Disable Synopsis section with version. + + root.DisableAutoGenTag = true + for _, c := range root.Commands() { + // Reduce update noise. + c.DisableAutoGenTag = true + } + + if err := gen(root, outdir); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} diff --git a/internal/valtools/.gitignore b/internal/valtools/.gitignore new file mode 100644 index 00000000..d8f7cdcc --- /dev/null +++ b/internal/valtools/.gitignore @@ -0,0 +1,2 @@ +/bin/ +/output/ diff --git a/internal/valtools/LICENSE b/internal/valtools/LICENSE new file mode 100644 index 00000000..0363628a --- /dev/null +++ b/internal/valtools/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2024 Metrum Research Group + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/internal/valtools/README.md b/internal/valtools/README.md new file mode 100644 index 00000000..b8f0405f --- /dev/null +++ b/internal/valtools/README.md @@ -0,0 +1,264 @@ + +Tools for validating Go executables +=================================== + + +Overview +-------- + +As part of validating a release of a package, MetrumRG produces two +artifacts, the source code archive and a scorecard report. For non-R +packages, the [mpn.scorecard][ms] R package takes care of *rendering* +the scorecard report. + +[ms]: https://github.com/metrumresearchgroup/mpn.scorecard + +This repository includes Go commands, shell scripts, and Makefile +rules that Go modules can use to generate the source code archive and +the inputs for mpn.scorecard. Its current focus is on validating +functionality exposed via **command-line executables**. + + +Key components +-------------- + + * [rules.mk][]: a Makefile that defines a `vt-all` target that + generates the source code archive and mpn.scorecard inputs + + * [checkmat/][]: Go package that defines an executable for running + various checks on a traceability matrix YAML file + + * [filecov/][]: Go package that defines an executable for calculating + *file*-level coverage from a Go coverage profile + + * [fmttests/][]: Go package that defines an executable for converting + `go test -json` into a custom format meant for inclusion in the + scorecard + + * [scripts/][]: various shell scripts invoked by targets in + [rules.mk][] + +[rules.mk]: ./rules.mk +[checkmat/]: ./checkmat +[filecov/]: ./filecov +[fmttests/]: ./fmttests +[scripts/]: ./scripts/ + + +Setup instructions +------------------ + +High-level summary of the main steps (details in subsections below): + + 1. add this repository as a subtree in the main repository + + 2. add documentation file for each command + + 3. create a traceability matrix YAML + + 4. add scripts for running the tests + + 5. include this subtree's `rules.mk` into the top-level Makefile + +### 1. Add subtree + +This repository is designed to be incorporated as a subtree in the +main project. The suggested location is `internal/valtools/`. + +You can use [git subtree][gs] to manage the subtree. + +[gs]: https://manpages.debian.org/stable/git-man/git-subtree.1.en.html + +Run `go mod tidy` to update `go.mod` with new dependencies, if any, +brought in by the subtree. + +### 2. Command documentation + +Consider a Go module that defines one executable, `foo`, where the +user-facing functionality is exposed via two subcommands, `foo bar` +and `foo baz`. In order to define the traceability matrix, there must +be a directory that contains a documentation file for each of these +commands, replacing any spaces in the name with underscores. + + docs/commands/ + |-- foo.md + |-- foo_bar.md + `-- foo_baz.md + +`docs/commands/` is taken as the directory for the command +documentation unless the `VT_DOC_DIR` variable specifies another +location (see [step 5](#step5)). + +By default, the Makefile rules expect the main repository to define a +`docgen` package (suggested location: `internal/tools/docgen`) whose +executable generates the documentation. + +The `docgen` executable should accept one argument, the directory in +which to write the documentation files. The executable is responsible +for ensuring that no stale documentation remains in the directory +(e.g., by removing the directory before writing new files). + +If a module uses `cobra`, `docgen` can likely be defined as a light +wrapper around the `cobra/doc.GenMarkdownTree` function. + +If the documentation files are maintained as the primary source +(i.e. the files do not need to be generated), set the `VT_DOC_DO_GEN` +variable to `no` (see [step 5](#step5)). + +### 3. Define traceability matrix + +Create a traceability matrix YAML file that maps each command to the +code file that defines it, the file that documents it, and the main +files that test it. By default, the Makefile rules look for the +matrix YAML file at `docs/validation/matrix.yaml`. To change this +location, set the `VT_MATRIX` variable (see [step 5](#step5)). + +The file should consist of a sequence of entries with the following +items: + + * `entrypoint`: the name of the command, as invoked by the user + + * `code`: the path to where the command is defined + + * `doc`: path to the command's main documentation + + * `tests`: list of paths where the command is tested + +Example entry: + + - entrypoint: foo bar + code: cmd/bar.go + doc: docs/commands/foo_bar.md + tests: + - cmd/bar_test.go + - integration/bar_test.go + +The `checkmat` tool will flag any documentation file that does not map +to a command with an entry in the YAML file. In some cases, you may +not want to include a command in the rendered matrix. For example, +the top-level `foo` command may serve only as an entry point for +subcommands, making it "uninteresting" to include in the matrix. For +such cases, you can add a skip entry to the matrix. + + - entrypoint: foo + skip: true + +### 4. Add test runners + +The main repository must define one or more scripts for running its +tests. Specify the paths to these scripts via the `VT_TEST_RUNNERS` +variable (see [step 5](#step5)). + +A test script must + + * write `go test` JSON records to standard output when passed the + `-json` argument. It must not write anything else to standard + output in this case. + + * check whether the `GOCOVERDIR` environment variable is set and, if + so, instrument the tests to write coverage data files under it. + +How to handle the `GOCOVERDIR` environment variable is determined by +whether tests are integration tests that use a built executable or +unit tests. + + * integration tests: when `GOCOVERDIR` is set, the script should pass + `-cover` to the `go build` call to build the instrumented + executable(s) to test + + * unit tests: when `GOCOVERDIR` is set, `go test` should be + instructed to write coverage data files to `GOCOVERDIR`. This can + be done by adding `-args -test.gocoverdir="$GOCOVERDIR"` to the end + of the call. + + When tallying coverage, Go does not by default count a statement as + covered if it's only executed via another package's unit tests. To + change that, list all the module's packages of interest by + specifying `-coverpkg` in the `go test` call. + +Notes: + + * Go 1.20 [introduced support][newcov] for `GOCOVERDIR` and + instrumenting executables. + + * There's a [proposed patch][covarg] to expose `test.gocoverdir` as + top-level argument to `go test`, although it's currently on hold. + +[newcov]: https://go.dev/blog/integration-test-coverage +[covarg]: https://go-review.googlesource.com/c/go/+/456595/14 + + + + +### 5. Wire up Makefile + +To wire up the subtree to the main repository, include the subtree's +`rules.mk` file in the repository's top-level Makefile. Before the +`include` directive, you can specify any Makefile [variables](#vars), +but, at a minimum, you should set `VT_TEST_RUNNERS`. + + VT_TEST_RUNNERS = scripts/run-unit-tests + VT_TEST_RUNNERS += scripts/run-integration-tests + include internal/valtools/rules.mk + + +Running the pipeline +-------------------- + +The `vt-all` target provides the main entry point. It generates the +source archive and mpn.scorecard inputs. + + make vt-all + +The generated files are written under the directory specified by the +variable `VT_OUT_DIR`. By default, this points to +`{subtree}/output/{package}_{version}`. + + + +### Makefile variables + + * `VT_BIN_DIR`: where to install executables (default: + `{subtree}/bin`) + + * `VT_DOC_DIR`: tell `docgen` executable to generate documentation + files under this directory (default: `docs/commands`) + + * `VT_DOC_DO_GEN`: whether to run `docgen` executable to generate + documentation files under `VT_DOC_DIR` (default: `yes`) + + * `VT_MATRIX`: path to matrix file (default: + `docs/validation/matrix.yaml`) + + * `VT_OUT_DIR`: where to generate the results (default: + `{subtree}/output/{package}_{version}`) + + * `VT_PKG`: name of the package (default: the base name of the + top-level directory). + + * `VT_TEST_ALLOW_SKIPS`: whether to allow skips when running the + `VT_TEST_RUNNERS` scripts (default: `no`) + + * `VT_TEST_RUNNERS`: a space-delimited list of scripts to invoke to + run the test suite + +### Auxiliary targets + +In addition to `vt-all`, the following targets can be useful to run +directly: + + * `vt-gen-docs`: invoke the `docgen` executable to refresh the + documentation in `VT_DOC_DIR` + + * `vt-cover-unlisted`: display a diff of two file sets: 1) non-test + Go files in the repository that define at least one function and 2) + files included in the coverage JSON under `VT_OUT_DIR` + + This can help identify files that are unexpectedly missing coverage + scores. In this case, you may need to adjust the package selection + or the `-coverpkg` value in one of the test runners scripts. + + * `vt-test`: invoke each script in `VT_TEST_RUNNERS` *without* + coverage enabled + +Run `vt-help` target to see a more complete list of targets. diff --git a/internal/valtools/checkmat/checkmat_test.go b/internal/valtools/checkmat/checkmat_test.go new file mode 100644 index 00000000..77799548 --- /dev/null +++ b/internal/valtools/checkmat/checkmat_test.go @@ -0,0 +1,609 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func assertCode(t *testing.T, output string, code string, ntimes int) { + t.Helper() + found := strings.Count(output, "["+code+"]") + if found != ntimes { + t.Errorf("expected %dx [%s] in output, got %d\noutput: %q", + ntimes, code, found, output) + } +} + +func TestCheckValidFileNamesBad(t *testing.T) { + var tests = []struct { + name string + entries []entry + want int + }{ + { + name: "one entry", + entries: []entry{ + { + Entrypoint: "foo", + Doc: "/foo/bar", + Tests: []string{"foo/../bar", "/baz"}, + }, + }, + want: 3, + }, + { + name: "multiple entries", + entries: []entry{ + { + Entrypoint: "a", + Doc: "/foo/bar", + Tests: []string{"foo/../bar"}, + }, + { + Entrypoint: "b", + Doc: "./foo/bar", + }, + { + Entrypoint: "c", + Doc: "good", + }, + }, + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + bad, err := checkValidFileNames(tt.entries, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != tt.want { + t.Errorf("invalid file names: want %d, got %d", tt.want, bad) + } + out := buf.String() + assertCode(t, out, "01", tt.want) + }) + } +} + +func TestCheckValidFileNamesGood(t *testing.T) { + entries := []entry{ + { + Entrypoint: "a", + Doc: "foo/bar", + Tests: []string{"foo/bar_test.go"}, + }, + { + Entrypoint: "b", + Doc: "foo/bar", + }, + } + + var buf bytes.Buffer + bad, err := checkValidFileNames(entries, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no invalid file names, got %d", bad) + } + + if out := buf.String(); out != "" { + t.Errorf("expected empty output, got %q", out) + } +} + +func TestCheckMissingFilesBad(t *testing.T) { + dir := t.TempDir() + var tests = []struct { + name string + entries []entry + want int + }{ + { + name: "one entry", + entries: []entry{ + { + Entrypoint: "foo", + Code: "cmd/foo.go", + Doc: "docs/commands/foo.md", + Tests: []string{"cmd/foo_test.go"}, + }, + }, + want: 3, + }, + { + name: "one entry without tests", + entries: []entry{ + { + Entrypoint: "foo", + Code: "cmd/foo.go", + Doc: "docs/commands/foo.md", + }, + }, + want: 2, + }, + { + name: "two entries", + entries: []entry{ + { + Entrypoint: "foo", + Code: "cmd/foo.go", + Doc: "docs/commands/foo.md", + Tests: []string{"cmd/foo_test.go"}, + }, + { + Entrypoint: "bar", + Code: "cmd/bar.go", + Doc: "docs/commands/bar.md", + Tests: []string{"cmd/bar_test.go", "another_test.go"}, + }, + }, + want: 7, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + bad, err := checkMissingFiles(tt.entries, dir, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != tt.want { + t.Errorf("missing files: want %d, got %d", tt.want, bad) + } + out := buf.String() + assertCode(t, out, "02", tt.want) + }) + } +} + +func createEmptyFile(t *testing.T, name string) { + t.Helper() + fh, err := os.Create(name) + if err != nil { + t.Fatal(err) + } + err = fh.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestCheckMissingFilesGood(t *testing.T) { + dir := t.TempDir() + entries := []entry{ + { + Entrypoint: "foo", + Code: "cmd/foo.go", + Doc: "docs/foo.md", + Tests: []string{"cmd/foo_test.go"}, + }, + { + Entrypoint: "bar", + Code: "cmd/bar.go", + Doc: "docs/bar.md", + Tests: []string{"cmd/bar_test.go", "another_test.go"}, + }, + } + + err := os.MkdirAll(filepath.Join(dir, "cmd"), 0777) + if err != nil { + t.Fatal(err) + } + err = os.MkdirAll(filepath.Join(dir, "docs"), 0777) + if err != nil { + t.Fatal(err) + } + + files := []string{ + filepath.Join("cmd", "foo.go"), + filepath.Join("docs", "foo.md"), + filepath.Join("cmd", "foo_test.go"), + filepath.Join("cmd", "bar.go"), + filepath.Join("docs", "bar.md"), + filepath.Join("cmd", "bar_test.go"), + "another_test.go", + } + for _, f := range files { + createEmptyFile(t, filepath.Join(dir, f)) + } + + var buf bytes.Buffer + bad, err := checkMissingFiles(entries, dir, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no missing files, got %d", bad) + } + + if out := buf.String(); out != "" { + t.Errorf("expected empty output, got %q", out) + } +} + +func TestCheckEntrypointDocMismatchBad(t *testing.T) { + var tests = []struct { + name string + entries []entry + want int + }{ + { + name: "one entry", + entries: []entry{ + { + Entrypoint: "foo", + Doc: "docs/commands/bar.md", + }, + }, + want: 1, + }, + { + name: "two entries", + entries: []entry{ + { + Entrypoint: "foo", + Doc: "docs/commands/bar.md", + }, + { + Entrypoint: "bar", + Doc: "docs/commands/baz.md", + }, + }, + want: 2, + }, + { + name: "skip", + entries: []entry{ + { + Entrypoint: "foo", + Doc: "docs/commands/bar.md", + Skip: true, + }, + { + Entrypoint: "bar", + Doc: "docs/commands/baz.md", + }, + }, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + bad, err := checkEntrypointDocMismatch(tt.entries, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != tt.want { + t.Errorf("command/doc mismatches: want %d, got %d", tt.want, bad) + } + out := buf.String() + assertCode(t, out, "03", tt.want) + }) + } +} + +func TestCheckEntrypointDocMismatchGood(t *testing.T) { + entries := []entry{ + { + Entrypoint: "foo bar", + Doc: "docs/commands/foo_bar.md", + }, + { + Entrypoint: "baz", + Doc: "docs/commands/baz.md", + }, + } + + var buf bytes.Buffer + bad, err := checkEntrypointDocMismatch(entries, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no mismatches, got %d", bad) + } + + if out := buf.String(); out != "" { + t.Errorf("expected empty output, got %q", out) + } +} + +func TestCheckDupEntrypointsBad(t *testing.T) { + entries := []entry{ + { + Entrypoint: "foo", + }, + { + Entrypoint: "bar", + }, + { + Entrypoint: "foo", + }, + { + Entrypoint: "baz", + }, + } + + var buf bytes.Buffer + bad, err := checkDupEntrypoints(entries, &buf) + if err != nil { + t.Fatal(err) + } + + if wantBad := 1; bad != wantBad { + t.Errorf("expected %d duplicated entry, got %d", wantBad, bad) + } + out := buf.String() + assertCode(t, out, "04", 1) +} + +func TestCheckDupEntrypointsGood(t *testing.T) { + entries := []entry{ + { + Entrypoint: "foo", + }, + { + Entrypoint: "bar", + }, + { + Entrypoint: "baz", + }, + } + + var buf bytes.Buffer + bad, err := checkDupEntrypoints(entries, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no duplicated entries, got %d", bad) + } + + if out := buf.String(); out != "" { + t.Errorf("expected empty output, got %q", out) + } +} + +func TestCheckMissingEntriesBad(t *testing.T) { + dir := t.TempDir() + files := []string{ + "foo_bar.md", + "baz.md", + } + for _, f := range files { + fname := filepath.Join(dir, f) + createEmptyFile(t, fname) + } + + var tests = []struct { + name string + entries []entry + want int + }{ + { + name: "no entries", + entries: []entry{}, + want: 2, + }, + { + name: "one entry", + entries: []entry{ + { + Entrypoint: "foo bar", + }, + }, + want: 1, + }, + { + name: "other entry", + entries: []entry{ + { + Entrypoint: "baz", + }, + }, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + bad, err := checkMissingEntries(tt.entries, dir, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != tt.want { + t.Errorf("missing entries: want %d, got %d", tt.want, bad) + } + out := buf.String() + assertCode(t, out, "05", tt.want) + }) + } +} + +func TestCheckMissingEntriesGood(t *testing.T) { + dir := t.TempDir() + files := []string{ + "foo_bar.md", + "baz.md", + } + for _, f := range files { + fname := filepath.Join(dir, f) + createEmptyFile(t, fname) + } + + entries := []entry{ + { + Entrypoint: "foo bar", + }, + { + Entrypoint: "baz", + }, + } + + var buf bytes.Buffer + bad, err := checkMissingEntries(entries, dir, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no missing entries, got %d", bad) + } + + if out := buf.String(); out != "" { + t.Errorf("expected empty output, got %q", out) + } +} + +func writeEntries(t *testing.T, es []entry, outfile string) { + t.Helper() + bs, err := yaml.Marshal(&es) + if err != nil { + t.Fatal(err) + } + fd, err := os.Create(outfile) + if err != nil { + fd.Close() + t.Fatal(err) + } + _, err = fd.Write(bs) + if err != nil { + t.Fatal(err) + } + + err = fd.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestCheckAll(t *testing.T) { + dir := t.TempDir() + docdir := filepath.Join(dir, "docs", "commands") + err := os.MkdirAll(docdir, 0777) + if err != nil { + t.Fatal(err) + } + + codedir := filepath.Join(dir, "cmd") + err = os.MkdirAll(codedir, 0777) + if err != nil { + t.Fatal(err) + } + + files := []string{ + filepath.Join(docdir, "foo_bar.md"), + filepath.Join(docdir, "baz.md"), + filepath.Join(docdir, "skip.md"), + filepath.Join(codedir, "foobar.go"), + filepath.Join(codedir, "baz.go"), + filepath.Join(codedir, "baz_test.go"), + } + for _, f := range files { + createEmptyFile(t, f) + } + + goodEntries := []entry{ + { + Entrypoint: "foo bar", + Code: "cmd/foobar.go", + Doc: "docs/commands/foo_bar.md", + }, + { + Entrypoint: "baz", + Code: "cmd/baz.go", + Doc: "docs/commands/baz.md", + Tests: []string{"cmd/baz_test.go"}, + }, + { + Entrypoint: "skip", + Skip: true, + }, + } + + yfile := filepath.Join(dir, "docs", "matrix.yaml") + writeEntries(t, goodEntries, yfile) + + t.Run("all good", func(t *testing.T) { + var buf bytes.Buffer + bad, err := check(yfile, docdir, dir, &buf) + if err != nil { + t.Fatal(err) + } + + if bad != 0 { + t.Errorf("expected no missing entries, got %d", bad) + } + + out := buf.String() + if out != "" { + t.Errorf("expected empty output, got %q", out) + } + }) + + // Now force each category of failure. + + badEntries := []entry{ + // Non-existent files (02), command/doc mismatch (03). + { + Entrypoint: "notthere", + Code: "cmd/../notthere.go", + Doc: "docs/commands/nt.md", + Tests: []string{"cmd/notthere_test.go"}, + }, + // Repeated entry (04). + { + Entrypoint: "foo bar", + Code: "cmd/foobar.go", + Doc: "docs/commands/foo_bar.md", + }, + } + + writeEntries(t, append(goodEntries, badEntries...), yfile) + // Documentation file without entry (05). + createEmptyFile(t, filepath.Join(docdir, "noentry.md")) + + t.Run("some bad", func(t *testing.T) { + var buf bytes.Buffer + bad, err := check(yfile, docdir, dir, &buf) + if err != nil { + t.Fatal(err) + } + + wantBad := 7 + if bad != wantBad { + t.Errorf("expected %d missing entries, got %d", wantBad, bad) + } + + out := buf.String() + assertCode(t, out, "01", 1) + assertCode(t, out, "02", 3) + assertCode(t, out, "03", 1) + assertCode(t, out, "04", 1) + assertCode(t, out, "05", 1) + }) +} diff --git a/internal/valtools/checkmat/main.go b/internal/valtools/checkmat/main.go new file mode 100644 index 00000000..f0dd98fa --- /dev/null +++ b/internal/valtools/checkmat/main.go @@ -0,0 +1,265 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +const usageMessage = `usage: checkmat + +Run various checks on the traceability matrix defined in . Each file in + is taken as the documentation for an entry point, where the base name maps +to an entry point in once the file extension is removed and underscores +are substituted for spaces. + +Verify that + + [01] each file name is valid + + To be considered "valid", a file name must be relative, must use "/" as + the path separator, and must not contain "." or ".." elements. + + [02] each named file exists + + The current working directory is taken as the top-level project directory, + and the files should be relative to this. + + [03] the base file name for the documentation matches the entry point name + + [04] no entry point has more than one entry in + + [05] for each file in , an entry with a matching entry point is found in + + + This check is skipped for any entries with a true "skip" value. + +Exit with status 1 if any issues are found. +` + +func usage() { + fmt.Fprint(flag.CommandLine.Output(), usageMessage) +} + +type entry struct { + Entrypoint string + Code string + Doc string + Tests []string + Skip bool +} + +func readEntries(f string) ([]entry, error) { + bs, err := os.ReadFile(f) + if err != nil { + return nil, err + } + var es []entry + if err := yaml.Unmarshal(bs, &es); err != nil { + return nil, err + } + + return es, nil +} + +func entryFiles(e entry) []string { + var fs []string + if e.Code != "" { + fs = append(fs, e.Code) + } + if e.Doc != "" { + fs = append(fs, e.Doc) + } + fs = append(fs, e.Tests...) + + return fs +} + +func checkValidFileNames(es []entry, w io.Writer) (int, error) { + var bad int + + for _, e := range es { + for _, f := range entryFiles(e) { + if !fs.ValidPath(f) { + bad++ + fmt.Fprintln(w, "[01] file name is invalid:", f) + } + } + } + + return bad, nil +} + +func checkMissingFiles(es []entry, topdir string, w io.Writer) (int, error) { + var bad int + + for _, e := range es { + for _, f := range entryFiles(e) { + // Note: Join() takes care of replacing slashes with + // os.PathSeparator. + f = filepath.Join(topdir, f) + _, err := os.Stat(f) + if errors.Is(err, os.ErrNotExist) { + bad++ + fmt.Fprintln(w, "[02] file does not exist:", f) + } else if err != nil { + return bad, err + } + } + } + + return bad, nil +} + +// TODO: Make it possible to customize how to map the doc file to the entry +// point name. As is it is only useful for command-line subcommands or one-off +// top-level commands, and it assumes that none of the commands have an +// underscore in their name. + +func docToEntrypoint(f string) string { + base := filepath.Base(f) + name := strings.TrimSuffix(base, filepath.Ext(base)) + + return strings.ReplaceAll(name, "_", " ") +} + +func checkEntrypointDocMismatch(es []entry, w io.Writer) (int, error) { + var bad int + + for _, e := range es { + if e.Skip { + continue + } + if docToEntrypoint(e.Doc) != e.Entrypoint { + bad++ + fmt.Fprintf(w, "[03] entry point and doc file mismatch: %q != %q\n", + e.Entrypoint, e.Doc) + } + } + + return bad, nil +} + +func checkDupEntrypoints(es []entry, w io.Writer) (int, error) { + var bad int + + cmds := make(map[string]bool) + for _, e := range es { + if _, found := cmds[e.Entrypoint]; found { + fmt.Fprintf(w, "[04] entry point %q defined more than once\n", + e.Entrypoint) + bad++ + } else { + cmds[e.Entrypoint] = true + } + } + + return bad, nil +} + +func checkMissingEntries(es []entry, docdir string, w io.Writer) (int, error) { + var bad int + + fh, err := os.Open(docdir) + if err != nil { + return bad, err + } + defer fh.Close() + + fnames, err := fh.Readdirnames(-1) + if err != nil { + return bad, err + } + + cmds := make(map[string]bool) + for _, e := range es { + cmds[e.Entrypoint] = true + } + + for _, f := range fnames { + if _, found := cmds[docToEntrypoint(f)]; !found { + fmt.Fprintf(w, "[05] No yaml entry for %q\n", filepath.Join(docdir, f)) + bad++ + } + } + + return bad, nil +} + +// check runs all the check functions on the traceability matrix defined in file +// yaml and returns the total number of issues found. docdir points to a +// directory containing the documentation files. topdir is an absolute path to +// top-level directory to which files in `yaml` are specified as relative. +// +// For each issue found, a message is written to w. +func check(yaml string, docdir string, topdir string, w io.Writer) (int, error) { + var bad int + + entries, err := readEntries(yaml) + if err != nil { + return bad, err + } + + type check func([]entry, io.Writer) (int, error) + checks := []check{ + checkValidFileNames, + func(es []entry, w io.Writer) (int, error) { + return checkMissingFiles(es, topdir, w) + }, + checkEntrypointDocMismatch, + checkDupEntrypoints, + func(es []entry, w io.Writer) (int, error) { + return checkMissingEntries(es, docdir, w) + }, + } + + for _, f := range checks { + n, err := f(entries, w) + if err != nil { + return bad, err + } + bad += n + } + + return bad, nil +} + +func main() { + if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help") { + flag.CommandLine.SetOutput(os.Stdout) + } + flag.Usage = usage + flag.Parse() + args := flag.Args() + if len(args) != 2 { + flag.Usage() + os.Exit(2) + } + + wd, err := os.Getwd() + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(2) + } + + bad, err := check(args[0], args[1], wd, os.Stdout) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(2) + } + + if bad > 0 { + fmt.Printf("\nproblems found: %d\n", bad) + os.Exit(1) + } +} diff --git a/internal/valtools/filecov/filecov_test.go b/internal/valtools/filecov/filecov_test.go new file mode 100644 index 00000000..8e06e389 --- /dev/null +++ b/internal/valtools/filecov/filecov_test.go @@ -0,0 +1,353 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "bytes" + "encoding/json" + "math" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "golang.org/x/tools/cover" +) + +func assertNearEqual(t *testing.T, a float64, b float64) { + t.Helper() + if math.IsNaN(a) || math.IsNaN(b) { + t.Fatal("assertNearEqual: NaN values are not allowed") + } + + var tol float64 = 1e-8 + d := math.Abs(a - b) + if d >= tol { + t.Errorf("absolute difference exceeds tolerance (%e)\na=%f\nb=%f", tol, a, b) + } +} + +func assertFileCoverage(t *testing.T, got []*fileCoverage, want []*fileCoverage) { + t.Helper() + + ngot := len(got) + nwant := len(want) + if ngot != nwant { + t.Errorf("expected coverage for %d files, got %d", nwant, ngot) + + return + } + + mapWant := make(map[string]float64, nwant) + for _, fc := range want { + if _, found := mapWant[fc.File]; found { + t.Error("repeated coverage for", fc.File) + } + mapWant[fc.File] = fc.Coverage + } + + for _, fc := range got { + perc, found := mapWant[fc.File] + if found { + assertNearEqual(t, fc.Coverage, perc) + } else { + t.Error("no coverage measurement for", fc.File) + } + } +} + +// See cover.ParseProfilesFromReader for a description of the format. +var testProfile string = `mode: set +example.com/tmod/foo.go:3.16,5.2 1 1 +example.com/tmod/foo.go:7.21,9.2 1 0 +example.com/tmod/foo.go:11.21,13.2 1 0 +example.com/tmod/foo.go:15.19,17.2 1 0 +example.com/tmod/cmd/bar.go:3.18,5.2 1 1 +example.com/tmod/cmd/bar.go:7.25,9.2 1 0 +example.com/tmod/cmd/main.go:7.13,10.2 2 0 +` + +func TestPercentCovered(t *testing.T) { + r := strings.NewReader(testProfile) + profiles, err := cover.ParseProfilesFromReader(r) + if err != nil { + t.Fatal(err) + } + cov := percentCovered(profiles) + assertNearEqual(t, cov.Overall, 25.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "example.com/tmod/foo.go", + Coverage: 25.0, + }, + { + File: "example.com/tmod/cmd/bar.go", + Coverage: 50.0, + }, + { + File: "example.com/tmod/cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} + +func TestPercentCoveredZero(t *testing.T) { + re := regexp.MustCompile(`(?m) 1$`) + r := strings.NewReader(re.ReplaceAllString(testProfile, " 0")) + profiles, err := cover.ParseProfilesFromReader(r) + if err != nil { + t.Fatal(err) + } + cov := percentCovered(profiles) + assertNearEqual(t, cov.Overall, 0.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "example.com/tmod/foo.go", + Coverage: 0.0, + }, + { + File: "example.com/tmod/cmd/bar.go", + Coverage: 0.0, + }, + { + File: "example.com/tmod/cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} + +func TestPercentCoveredNoFiles(t *testing.T) { + r := strings.NewReader("mode: set\n") + profiles, err := cover.ParseProfilesFromReader(r) + if err != nil { + t.Fatal(err) + } + cov := percentCovered(profiles) + assertNearEqual(t, cov.Overall, 0.0) + + if len(cov.Files) != 0 { + t.Errorf("expected coverage for 0 files, got %d", len(cov.Files)) + } +} + +func TestWrite(t *testing.T) { + r := strings.NewReader(testProfile) + profiles, err := cover.ParseProfilesFromReader(r) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = write(&buf, profiles, "", "") + if err != nil { + t.Fatal(err) + } + + var cov coverage + err = json.Unmarshal(buf.Bytes(), &cov) + if err != nil { + t.Fatal(err) + } + + assertNearEqual(t, cov.Overall, 25.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "example.com/tmod/foo.go", + Coverage: 25.0, + }, + { + File: "example.com/tmod/cmd/bar.go", + Coverage: 50.0, + }, + { + File: "example.com/tmod/cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} + +func TestWriteShortenNames(t *testing.T) { + r := strings.NewReader(testProfile) + profiles, err := cover.ParseProfilesFromReader(r) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = write(&buf, profiles, "example.com/tmod", "") + if err != nil { + t.Fatal(err) + } + + var cov coverage + err = json.Unmarshal(buf.Bytes(), &cov) + if err != nil { + t.Fatal(err) + } + + assertNearEqual(t, cov.Overall, 25.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "foo.go", + Coverage: 25.0, + }, + { + File: "cmd/bar.go", + Coverage: 50.0, + }, + { + File: "cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} + +func chdir(t *testing.T, dir string) { + old, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + err = os.Chdir(dir) + if err != nil { + _ = os.Chdir(old) + t.Fatal(err) + } + + t.Cleanup(func() { + _ = os.Chdir(old) + }) +} + +func setupRunDir(t *testing.T) string { + dir := t.TempDir() + realmodPath := filepath.Join(dir, "realmod") + err := os.MkdirAll(filepath.Join(realmodPath, "cmd"), 0777) + if err != nil { + t.Fatal(err) + } + + modPath := filepath.Join(dir, "mod") + err = os.Symlink(realmodPath, modPath) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(modPath, "go.mod"), + []byte("module example.com/tmod\n\ngo 1.22.5"), + 0666) + if err != nil { + t.Fatal(err) + } + + tfiles := []string{ + "foo.go", + "cmd/bar.go", + "cmd/main.go", + } + for _, f := range tfiles { + fh, err := os.Create(filepath.Join(modPath, f)) + if err != nil { + t.Fatal(err) + } + err = fh.Close() + if err != nil { + t.Fatal(err) + } + } + + chdir(t, modPath) + + return modPath +} + +func TestRun(t *testing.T) { + modPath := setupRunDir(t) + pcontent := strings.ReplaceAll(testProfile, + "example.com/tmod/cmd/main.go", + modPath+"/"+"cmd/main.go") + profPath := filepath.Join(modPath, "coverage.out") + err := os.WriteFile(profPath, []byte(pcontent), 0666) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = run(profPath, "go.mod", &buf) + if err != nil { + t.Fatal(err) + } + + var cov coverage + err = json.Unmarshal(buf.Bytes(), &cov) + if err != nil { + t.Fatal(err) + } + + assertNearEqual(t, cov.Overall, 25.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "foo.go", + Coverage: 25.0, + }, + { + File: "cmd/bar.go", + Coverage: 50.0, + }, + { + File: "cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} + +func TestRunNoGoMod(t *testing.T) { + modPath := setupRunDir(t) + profPath := filepath.Join(modPath, "coverage.out") + err := os.WriteFile(profPath, []byte(testProfile), 0666) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err = run(profPath, "", &buf) + if err != nil { + t.Fatal(err) + } + + var cov coverage + err = json.Unmarshal(buf.Bytes(), &cov) + if err != nil { + t.Fatal(err) + } + + assertNearEqual(t, cov.Overall, 25.0) + assertFileCoverage(t, cov.Files, + []*fileCoverage{ + { + File: "example.com/tmod/foo.go", + Coverage: 25.0, + }, + { + File: "example.com/tmod/cmd/bar.go", + Coverage: 50.0, + }, + { + File: "example.com/tmod/cmd/main.go", + Coverage: 0.0, + }, + }, + ) +} diff --git a/internal/valtools/filecov/main.go b/internal/valtools/filecov/main.go new file mode 100644 index 00000000..77f449b0 --- /dev/null +++ b/internal/valtools/filecov/main.go @@ -0,0 +1,201 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/tools/cover" +) + +const usageMessage = `usage: filecov [-mod=] + +Calculate the coverage for each file in along with the overall +coverage and write the JSON result to standard output. is a coverage +profile in the format generated by 'go test' for its -coverprofile argument. + +Options: + + -mod= + The file entries in are prefixed with the module name (e.g., + "example.com/mod/cmd/foo.go") or, in some cases, the absolute path to the + file. If you pass a module's go.mod to this option, the module name from + that file is stripped from the file names in the output (yielding, e.g., + "cmd/foo.go"). This is particularly useful in the common case where all the + files belong to the same module. ` + +var ( + gomod = flag.String("mod", "", "") +) + +func usage() { + fmt.Fprint(flag.CommandLine.Output(), usageMessage) +} + +type tally struct { + covered int64 + total int64 +} + +func tallyCovered(p *cover.Profile) tally { + // This follows Go's src/cmd/cover/html.go. + var total, covered int64 + for _, b := range p.Blocks { + total += int64(b.NumStmt) + if b.Count > 0 { + covered += int64(b.NumStmt) + } + } + + return tally{covered: covered, total: total} +} + +type fileCoverage struct { + File string `json:"file"` + Coverage float64 `json:"coverage"` +} + +type coverage struct { + Overall float64 `json:"overall"` + Files []*fileCoverage `json:"files"` +} + +func percent(covered int64, total int64) float64 { + if total == 0 { + return 0 + } + + return float64(covered) / float64(total) * 100 +} + +func percentCovered(profiles []*cover.Profile) coverage { + var total, covered int64 + var fcovs []*fileCoverage + + for _, profile := range profiles { + ftally := tallyCovered(profile) + fcovs = append(fcovs, + &fileCoverage{ + File: profile.FileName, + Coverage: percent(ftally.covered, ftally.total), + }, + ) + covered += ftally.covered + total += ftally.total + } + + return coverage{Overall: percent(covered, total), Files: fcovs} +} + +func shortenFileNames(cov coverage, modpath string, localpath string) error { + if modpath == "" { + return nil + } + + mprefix := modpath + "/" + sep := string(filepath.Separator) + + var lprefix string + if localpath != "" { + lpath, err := filepath.EvalSymlinks(localpath) + if err != nil { + return err + } + lprefix = lpath + string(filepath.Separator) + } + + for _, entry := range cov.Files { + var prefix, file string + var err error + if lprefix != "" && strings.HasPrefix(entry.File, sep) { + // For the executable, Go writes a path like + // /path/to/bbi/cmd/bbi/main.go instead of the module-prefixed name + // (github.com/metrumresearchgroup/bbi/...) that it writes for the + // other files. + file, err = filepath.EvalSymlinks(entry.File) + if err != nil { + return err + } + prefix = lprefix + } else { + file = entry.File + prefix = mprefix + } + entry.File = strings.TrimPrefix(file, prefix) + } + + return nil +} + +func write(w io.Writer, profiles []*cover.Profile, modpath string, localpath string) error { + cov := percentCovered(profiles) + if modpath != "" { + if err := shortenFileNames(cov, modpath, localpath); err != nil { + return err + } + } + + bs, err := json.MarshalIndent(cov, "", " ") + if err != nil { + return err + } + _, err = w.Write(append(bs, []byte("\n")...)) + + return err +} + +func run(input string, gomod string, w io.Writer) error { + profiles, err := cover.ParseProfiles(input) + if err != nil { + return err + } + + var mpath, lpath string + if gomod != "" { + gomod, err = filepath.Abs(gomod) + if err != nil { + return err + } + + mod, err := os.ReadFile(gomod) + if err != nil { + return err + } + + mpath = modfile.ModulePath(mod) + if mpath == "" { + return fmt.Errorf("could not find module path in %q", gomod) + } + + lpath = filepath.Dir(gomod) + } + + return write(w, profiles, mpath, lpath) +} + +func main() { + if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help") { + flag.CommandLine.SetOutput(os.Stdout) + } + flag.Usage = usage + flag.Parse() + args := flag.Args() + + if len(args) != 1 { + flag.Usage() + os.Exit(2) + } + + if err := run(args[0], *gomod, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(2) + } +} diff --git a/internal/valtools/fmttests/fmttests_test.go b/internal/valtools/fmttests/fmttests_test.go new file mode 100644 index 00000000..fd2cd57f --- /dev/null +++ b/internal/valtools/fmttests/fmttests_test.go @@ -0,0 +1,400 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "bytes" + "encoding/json" + "io" + "strings" + "testing" + "time" +) + +type testEvent struct { + Action string + Package string + Test string + Output string +} + +func makeEventReader(t *testing.T, tes []testEvent) io.Reader { + t.Helper() + var buf bytes.Buffer + var e event + for _, te := range tes { + e = event{ + Time: time.Time{}, + Action: te.Action, + Package: te.Package, + Test: te.Test, + Output: te.Output, + } + bs, err := json.Marshal(&e) + if err != nil { + t.Fatal(err) + } + buf.Write(bs) + buf.Write([]byte("\n")) + } + + return bytes.NewReader(buf.Bytes()) +} + +func TestProcessEvents(t *testing.T) { + var tests = []struct { + name string + events []testEvent + subtests bool + lines []string + nfailed int + npassed int + nskipped int + }{ + { + name: "base", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + { + Action: "pass", + Package: "example.com/ghi/cmd", + Test: "TestBaz", + }, + }, + subtests: false, + lines: []string{ + "[def] TestFoo: passed", + "[cmd] TestBaz: passed", + }, + nfailed: 0, + npassed: 3, + nskipped: 0, + }, + { + name: "subtests via flag", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + { + Action: "pass", + Package: "example.com/ghi/cmd", + Test: "TestBaz", + }, + }, + subtests: true, + lines: []string{ + "[def] TestFoo: passed", + "[def] TestFoo/Bar: passed", + "[cmd] TestBaz: passed", + }, + nfailed: 0, + npassed: 3, + nskipped: 0, + }, + { + name: "include skipped subtests", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "skip", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + { + Action: "pass", + Package: "example.com/ghi/cmd", + Test: "TestBaz", + }, + }, + subtests: false, + lines: []string{ + "[def] TestFoo: passed", + "[def] TestFoo/Bar: skipped", + "[cmd] TestBaz: passed", + }, + nfailed: 0, + npassed: 2, + nskipped: 1, + }, + { + name: "no package", + events: []testEvent{ + { + Action: "pass", + Package: "", + Test: "TestFoo", + }, + }, + subtests: false, + lines: []string{ + "[] TestFoo: passed", + }, + nfailed: 0, + npassed: 1, + nskipped: 0, + }, + { + name: "include failed subtests", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "fail", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + { + Action: "pass", + Package: "example.com/ghi/cmd", + Test: "TestBaz", + }, + }, + subtests: false, + lines: []string{ + "[def] TestFoo: passed", + "[def] TestFoo/Bar: failed", + "[cmd] TestBaz: passed", + }, + nfailed: 1, + npassed: 2, + nskipped: 0, + }, + { + name: "filtered", + events: []testEvent{ + { + Action: "start", + Package: "example.com/abc/def", + }, + { + Action: "run", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + }, + subtests: false, + lines: []string{ + "[def] TestFoo: passed", + }, + nfailed: 0, + npassed: 1, + nskipped: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bufStdout bytes.Buffer + var bufStderr bytes.Buffer + s, err := processEvents(makeEventReader(t, tt.events), + tt.subtests, &bufStdout, &bufStderr) + if err != nil { + t.Fatal(err) + } + + if len(s.Failed) != tt.nfailed { + t.Errorf("failed: want %d, got %d", tt.nfailed, len(s.Failed)) + } + if len(s.Passed) != tt.npassed { + t.Errorf("passed: want %d, got %d", tt.npassed, len(s.Passed)) + } + if len(s.Skipped) != tt.nskipped { + t.Errorf("skipped: want %d, got %d", tt.nskipped, len(s.Skipped)) + } + + stdout := bufStdout.String() + stdoutWant := strings.Join(tt.lines, "\n") + "\n" + if stdout != stdoutWant { + t.Errorf("stdout:\n want %q,\n got %q", stdoutWant, stdout) + } + + stderr := bufStderr.String() + if stderr != "" { + t.Errorf("expected empty stderr, got %q", stderr) + } + }) + } +} + +func TestProcessEventsFailureOutput(t *testing.T) { + events := []testEvent{ + { + Action: "output", + Package: "example.com/o/foo", + Test: "TestFoo1", + Output: "foo1 output\n", + }, + { + Action: "pass", + Package: "example.com/o/foo", + Test: "TestFoo1", + }, + { + Action: "output", + Package: "example.com/o/foo", + Test: "TestFoo2", + Output: "TestFoo2 failed\n", + }, + { + Action: "output", + Package: "example.com/o/foo", + Test: "TestFoo2", + Output: "error was ...\n", + }, + { + Action: "fail", + Package: "example.com/o/foo", + Test: "TestFoo2", + }, + { + Action: "pass", + Package: "example.com/o/k/bar", + Test: "TestBar", + }, + { + Action: "output", + Package: "example.com/o/k/baz", + Test: "TestBaz/1", + Output: "TestBaz/1 failed\n", + }, + { + Action: "pass", + Package: "example.com/o/k/baz", + Test: "TestBaz/2", + }, + { + Action: "fail", + Package: "example.com/o/k/baz", + Test: "TestBaz/1", + }, + { + Action: "fail", + Package: "example.com/o/k/baz", + Test: "TestBaz", + }, + } + var bufStdout bytes.Buffer + var bufStderr bytes.Buffer + s, err := processEvents(makeEventReader(t, events), + false, &bufStdout, &bufStderr) + if err != nil { + t.Fatal(err) + } + + if nfailedWant := 3; len(s.Failed) != nfailedWant { + t.Errorf("failed: want %d, got %d", nfailedWant, len(s.Failed)) + } + + if npassedWant := 3; len(s.Passed) != npassedWant { + t.Errorf("passed: want %d, got %d", npassedWant, len(s.Passed)) + } + + if nskippedWant := 0; len(s.Skipped) != nskippedWant { + t.Errorf("skipped: want %d, got %d", nskippedWant, len(s.Skipped)) + } + + stdoutWant := strings.Join([]string{ + "[foo] TestFoo1: passed", + "[foo] TestFoo2: failed", + "[bar] TestBar: passed", + "[baz] TestBaz/1: failed", + "[baz] TestBaz: failed", + }, "\n") + "\n" + if stdout := bufStdout.String(); stdout != stdoutWant { + t.Errorf("stdout:\n want %q,\n got %q", stdoutWant, stdout) + } + + stderrWant := strings.Join([]string{ + "TestFoo2 failed", + "error was ...", + "TestBaz/1 failed", + }, "\n") + "\n" + if stderr := bufStderr.String(); stderr != stderrWant { + t.Errorf("stderr: want %q, got %q", stderrWant, stderr) + } +} + +func TestProcessEventsErrors(t *testing.T) { + var tests = []struct { + name string + events []testEvent + subtests bool + lines []string + nfailed int + npassed int + nskipped int + }{ + { + name: "bench", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "bench", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + }, + }, + { + name: "unknown", + events: []testEvent{ + { + Action: "pass", + Package: "example.com/abc/def", + Test: "TestFoo", + }, + { + Action: "unknown", + Package: "example.com/abc/def", + Test: "TestFoo/Bar", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bufStdout bytes.Buffer + var bufStderr bytes.Buffer + _, err := processEvents(makeEventReader(t, tt.events), + false, &bufStdout, &bufStderr) + if err == nil { + t.Errorf("processEvents unexpectedly passed") + } + }) + } +} diff --git a/internal/valtools/fmttests/main.go b/internal/valtools/fmttests/main.go new file mode 100644 index 00000000..274fde93 --- /dev/null +++ b/internal/valtools/fmttests/main.go @@ -0,0 +1,164 @@ +// Copyright 2024 Metrum Research Group +// SPDX-License-Identifier: MIT + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + "time" +) + +const usageMessage = `usage: fmttests [-allow-skips] [-subtests] + +Read 'go test -json' lines from standard input and display a formatted output +line for each test result record (action of "pass", "fail", or "skip"). + +Passing subtests are not displayed unless the -subtests flag is specified. +Failed or skipped subtests are always displayed. + +If any "fail" record is encountered, exit with status 1. Any "skip" record will +also trigger an exit with status 1 unless the -allow-skips flag is specified. +` + +var ( + subtests = flag.Bool("subtests", false, "") + allowSkips = flag.Bool("allow-skips", false, "") +) + +func usage() { + fmt.Fprint(flag.CommandLine.Output(), usageMessage) +} + +// Modified from Go's src/cmd/internal/test2json/test2json.go. +type event struct { + Time time.Time `json:",omitempty"` + Action string + Package string `json:",omitempty"` + Test string `json:",omitempty"` + Elapsed float64 `json:",omitempty"` + Output string `json:",omitempty"` +} + +type summary struct { + Failed []string + Passed []string + Skipped []string +} + +func packageBaseName(s string) string { + xs := strings.Split(s, "/") + n := len(xs) + if n == 0 { + return s + } + + return xs[n-1] +} + +type key struct { + Package string + Test string +} + +// processEvents reads a JSON record of a test event from r. For each result +// record encountered (i.e. a record with an action of "pass", "fail", or +// "skip"), it writes a formatted line to wout. If any failures are +// encountered, it writes the failing test's output lines to werr. +// +// The subtests argument controls whether results for subtests are written. +// +// processEvents returns a summary instance that records the test names for each +// result record encountered. +func processEvents(r io.Reader, subtests bool, wout io.Writer, werr io.Writer) (summary, error) { + var res summary + + failLines := make(map[key][]string) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + var e event + + if err := json.Unmarshal(scanner.Bytes(), &e); err != nil { + return res, err + } + if e.Test == "" { + continue + } + + var status string + var alwaysShow bool + switch e.Action { + case "fail": + k := key{e.Package, e.Test} + for _, line := range failLines[k] { + fmt.Fprint(werr, line) + } + res.Failed = append(res.Failed, e.Test) + status = "failed" + alwaysShow = true + case "pass": + res.Passed = append(res.Passed, e.Test) + status = "passed" + case "skip": + res.Skipped = append(res.Skipped, e.Test) + status = "skipped" + alwaysShow = true + case "bench": + return res, errors.New("bench output is not supported") + case "output": + k := key{e.Package, e.Test} + failLines[k] = append(failLines[k], e.Output) + + continue + case "cont", "pause", "run", "start": + continue + default: + return res, fmt.Errorf("unknown action: %s", e.Action) + } + + if subtests || alwaysShow || !strings.ContainsAny(e.Test, "/") { + // TODO: Consider other approaches for formatting the package name + // that avoid collisions (e.g., packageBaseName returns "cmd" for + // both ".../foo/cmd" and ".../bar/cmd"). + fmt.Fprintf(wout, "[%s] %s: %s\n", + packageBaseName(e.Package), e.Test, status) + } + } + + if err := scanner.Err(); err != nil { + return res, err + } + + return res, nil +} + +func main() { + if len(os.Args) > 1 && (os.Args[1] == "-h" || os.Args[1] == "--help") { + flag.CommandLine.SetOutput(os.Stdout) + } + flag.Usage = usage + flag.Parse() + + if flag.NArg() != 0 { + flag.Usage() + os.Exit(2) + } + + res, err := processEvents(os.Stdin, *subtests, os.Stdout, os.Stderr) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(2) + } + + if len(res.Failed) > 0 || (!*allowSkips && len(res.Skipped) > 0) { + fmt.Fprintf(os.Stderr, "failed tests: %d, skipped tests: %d\n", + len(res.Failed), len(res.Skipped)) + os.Exit(1) + } +} diff --git a/internal/valtools/rules.mk b/internal/valtools/rules.mk new file mode 100644 index 00000000..657fa527 --- /dev/null +++ b/internal/valtools/rules.mk @@ -0,0 +1,155 @@ +vtdir := $(dir $(lastword $(MAKEFILE_LIST))) +vtdir := $(patsubst %/,%,$(vtdir)) +ifeq ($(strip $(vtdir)),) +$(error "bug: vtdir is unexpectedly empty") +endif + +ifeq ($(VT_PKG),) +VT_PKG := $(notdir $(CURDIR)) +endif + +version := $(shell git describe --tags --always HEAD) +version := $(version:v%=%) +name := $(VT_PKG)_$(version) + +VT_OUT_DIR ?= $(vtdir)/output/$(name) +prefix := $(VT_OUT_DIR)/$(name) + +VT_BIN_DIR ?= $(vtdir)/bin +VT_DOC_DIR ?= docs/commands +VT_DOC_DO_GEN ?= yes +VT_MATRIX ?= docs/validation/matrix.yaml + +VT_TEST_ALLOW_SKIPS ?= no +VT_TEST_RUNNERS ?= +ifeq ($(strip $(VT_TEST_RUNNERS)),) +$(error "VT_TEST_RUNNERS must point to space-delimited list of test scripts") +endif + +.PHONY: vt-help +vt-help: + $(info Primary targets:) + $(info * vt-all: create all validation artifacts under $(VT_OUT_DIR)/) + $(info * vt-gen-docs: generate command docs under $(VT_DOC_DIR)/) + $(info ) + $(info Other targets:) + $(info * vt-cover-unlisted: show Go files that are not in coverage JSON) + $(info * vt-test: invoke each script listed in VT_TEST_RUNNERS) + $(info ) + $(info Other targets, triggered by vt-all:) + $(info * vt-archive: write source archive to $(prefix).tar.gz) + $(info * vt-bin: install executables for packages under current directory to $(VT_BIN_DIR)/) + $(info * vt-checkmat: check $(VT_MATRIX) with checkmat) + $(info * vt-copymat: copy $(VT_MATRIX) to $(prefix).matrix.yaml) + $(info * vt-cover: invoke each script listed in VT_TEST_RUNNERS with coverage enabled) + $(info * vt-metadata: write scorecard metadata to $(prefix).metadata.json) + $(info * vt-pkg: write package name and version to $(prefix).pkg.json) + $(info * vt-scores: write scorecard scores to $(prefix).scores.json) + @: + +.PHONY: help-valtools +help-valtools: vt-help + +.PHONY: vt-all +vt-all: vt-copymat +vt-all: vt-cover +vt-all: vt-scores +vt-all: vt-pkg +vt-all: vt-metadata +vt-all: vt-archive + +.PHONY: vt-bin +vt-bin: + rm -rf '$(VT_BIN_DIR)' && mkdir '$(VT_BIN_DIR)' + go build -o '$(VT_BIN_DIR)/' ./... + +.PHONY: vt-gen-docs +vt-gen-docs: +ifeq ($(VT_DOC_DO_GEN),yes) + $(MAKE) vt-bin + @test -f '$(VT_BIN_DIR)/docgen' || \ + { printf '"make vt-bin" did not generate $(VT_BIN_DIR)/docgen\n'; exit 1; } + '$(VT_BIN_DIR)/docgen' '$(VT_DOC_DIR)' +else + @: +endif + +$(VT_BIN_DIR)/checkmat: $(vtdir)/checkmat/main.go + +.PHONY: vt-checkmat +vt-checkmat: $(VT_BIN_DIR)/checkmat + '$(VT_BIN_DIR)/checkmat' '$(VT_MATRIX)' '$(VT_DOC_DIR)' + +.PHONY: vt-copymat +vt-copymat: + $(MAKE) vt-gen-docs + $(MAKE) vt-checkmat + @test -z "$$(git status -unormal --porcelain -- '$(VT_DOC_DIR)')" || \ + { printf 'commit changes to $(VT_DOC_DIR) first\n'; exit 1; } + @mkdir -p '$(VT_OUT_DIR)' + cp '$(VT_MATRIX)' '$(prefix).matrix.yaml' + +$(VT_BIN_DIR)/fmttests: $(vtdir)/fmttests/main.go + +.PHONY: vt-test +vt-test: $(VT_BIN_DIR)/fmttests + @unset GOCOVERDIR; \ + '$(vtdir)/scripts/run-tests' '$(VT_BIN_DIR)/fmttests' \ + '$(VT_TEST_ALLOW_SKIPS)' $(VT_TEST_RUNNERS) + +$(VT_BIN_DIR)/filecov: $(vtdir)/filecov/main.go + +# ATTN: Make coverage directory absolute because we cannot rely on +# test subprocesses to be executed from the same directory. +cov_dir := $(abspath $(VT_OUT_DIR)/.coverage) +cov_prof := $(cov_dir).profile + +.PHONY: vt-cover +vt-cover: export GOCOVERDIR=$(cov_dir) +vt-cover: $(VT_BIN_DIR)/filecov +vt-cover: $(VT_BIN_DIR)/fmttests + @mkdir -p '$(VT_OUT_DIR)' + rm -rf '$(cov_dir)' && mkdir '$(cov_dir)' + '$(vtdir)/scripts/run-tests' '$(VT_BIN_DIR)/fmttests' \ + '$(VT_TEST_ALLOW_SKIPS)' $(VT_TEST_RUNNERS) \ + >'$(prefix).check.txt' + go tool covdata textfmt -i '$(cov_dir)' -o '$(cov_prof)' + '$(VT_BIN_DIR)/filecov' -mod go.mod '$(cov_prof)' \ + >'$(prefix).coverage.json' + +.PHONY: vt-cover-unlisted +vt-cover-unlisted: + @test -f '$(prefix).coverage.json' || \ + { printf >&2 'vt-cover-unlisted requires $(prefix).coverage.json\n'; exit 1; } + @'$(vtdir)/scripts/cover-unlisted' '$(prefix).coverage.json' || : + +.PHONY: vt-scores +vt-scores: + @mkdir -p '$(VT_OUT_DIR)' + '$(vtdir)/scripts/write-scores' '$(prefix).coverage.json' \ + >'$(prefix).scores.json' + +.PHONY: vt-pkg +vt-pkg: + @mkdir -p '$(VT_OUT_DIR)' + jq -n --arg p '$(VT_PKG)' --arg v "$(version)" \ + '{"mpn_scorecard_format": "1.0",'\ + ' "pkg_name": $$p, "pkg_version": $$v,'\ + ' "scorecard_type": "cli"}' \ + >'$(prefix).pkg.json' + +.PHONY: vt-metadata +vt-metadata: + @mkdir -p '$(VT_OUT_DIR)' + '$(vtdir)/scripts/metadata' >'$(prefix).metadata.json' + +.PHONY: vt-archive +vt-archive: + @mkdir -p '$(VT_OUT_DIR)' + @test -z "$(git status --porcelain -unormal --ignore-submodules=none)" || \ + { printf >&2 'working tree is dirty; commit changes first\n'; exit 1; } + git archive -o '$(prefix).tar.gz' --format=tar.gz HEAD + +$(VT_BIN_DIR)/%: $(vtdir)/%/main.go + @mkdir -p '$(VT_BIN_DIR)' + go build -o '$@' '$<' diff --git a/internal/valtools/scripts/cover-unlisted b/internal/valtools/scripts/cover-unlisted new file mode 100755 index 00000000..3ba837cd --- /dev/null +++ b/internal/valtools/scripts/cover-unlisted @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright 2024 Metrum Research Group +# SPDX-License-Identifier: MIT + +set -eu + +test $# = 1 || { + printf >&2 'usage: %s \n' "$0" + exit 1 +} + +tdir=$(mktemp -d "${TMPDIR:-/tmp}"/valtools-XXXXX) +trap 'rm -rf "$tdir"' 0 + +jq -r '.files | .[] | .file' <"$1" | sort >"$tdir"/files-in-coverage + +# Limit to files with a function definition because Go, by design, +# does not consider those. See Go's d1cb5c0605 (cmd/go: improve +# handling of no-test packages for coverage, 2023-05-09). +git grep -l --full-name '^func ' ':(top)*.go' | \ + grep -Ev '_test.go$' | \ + sort >"$tdir"/files-in-tree + +git diff --no-index "$tdir"/files-in-tree "$tdir"/files-in-coverage diff --git a/internal/valtools/scripts/metadata b/internal/valtools/scripts/metadata new file mode 100755 index 00000000..92d1f6c5 --- /dev/null +++ b/internal/valtools/scripts/metadata @@ -0,0 +1,48 @@ +#!/bin/sh +# Copyright 2024 Metrum Research Group +# SPDX-License-Identifier: MIT + +set -eu + +test $# = 0 || { + printf >&2 'usage: %s\n' "$0" + exit 1 +} + +# Note: This output matches the "metadata" field that mpn.scorecard +# outputs to the .scorecard.json for R packages, with the addition of +# the Go version. + +dt=$(date '+%Y-%m-%d %H:%M:%S') +user=${USER?'USER environment variable is not set'} +sysname=$(uname -s) +version=$(uname -v) +release=$(uname -r) +machine=$(uname -m) +mver=${METWORX_VERSION?'METWORX_VERSION environment variable is not set'} + +go_ver=$(go env GOVERSION) +go_ver=${go_ver#go} + +jq -n \ + --arg d "$dt" \ + --arg u "$user" \ + --arg s "$sysname" \ + --arg v "$version" \ + --arg r "$release" \ + --arg m "$machine" \ + --arg V "$mver" \ + --arg g "$go_ver" \ + '{"date": $d, + "executor": $u, + "info": { + "env_vars": {"METWORX_VERSION": $V}, + "sys": { + "sysname": $s, + "version": $v, + "release": $r, + "machine": $m, + "Go version": $g + } + } + }' diff --git a/internal/valtools/scripts/run-tests b/internal/valtools/scripts/run-tests new file mode 100755 index 00000000..5733f740 --- /dev/null +++ b/internal/valtools/scripts/run-tests @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2024 Metrum Research Group +# SPDX-License-Identifier: MIT + +set -uo pipefail + +test $# -gt 2 || { + printf >&2 'usage: %s ...\n' "$0" + exit 1 +} + +fmt=$1 +shift +allow_skips=$(echo "$1" | tr '[:upper:]' '[:lower:]') +case "$allow_skips" in + 1|yes|y|true|t) + skip_arg=-allow-skips + ;; + *) + skip_arg= + ;; +esac +shift + +status=0 +for runner in "$@" +do + # shellcheck disable=SC2086 + "$runner" -json | "$fmt" $skip_arg || { + status=$? + printf >&2 '%s failed\n' "$runner" + } +done +exit "$status" diff --git a/internal/valtools/scripts/write-scores b/internal/valtools/scripts/write-scores new file mode 100755 index 00000000..396080dc --- /dev/null +++ b/internal/valtools/scripts/write-scores @@ -0,0 +1,73 @@ +#!/bin/sh +# Copyright 2024 Metrum Research Group +# SPDX-License-Identifier: MIT +# +# shellcheck disable=SC3043 +# Note: shell must support non-POSIX 'local'. + +set -eu + +test $# = 1 || { + printf >&2 'usage: %s \n' "$0" + exit 1 +} + +covresults=$1 +cov=$(jq -e .overall <"$covresults") || { + printf >&2 'reading coverage from %s failed\n' "$covresults" + exit 1 +} + +ask () { + local ans + local res + + while true + do + printf >&2 '%s [yn] ' "$1" + read -r ans + case "$ans" in + y|Y|yes) + res=1 + break + ;; + n|N|no) + res=0 + break + ;; + *) + printf >&2 'Enter y or n.\n' + ;; + esac + done + printf '%s\n' "$res" +} + +# TODO: Check existence of NEWS.md (any others?) instead of prompting? +has_news=$(ask 'Does this package have a NEWS file?') +news_current=$(ask 'Does the version being scored have a NEWS entry?') +has_website=$(ask 'Does this package have a website?') + +jq -n \ + --argjson c "$cov" \ + --argjson w "$has_website" \ + --argjson n "$has_news" \ + --argjson N "$news_current" \ + '{ + "testing": { + "check": 1, + "coverage": ($c / 100) + }, + "documentation": { + "has_website": $w, + "has_news": $n + }, + "maintenance": { + "has_maintainer": 1, + "news_current": $N + }, + "transparency": { + "has_source_control": 1, + "has_bug_reports_url": 1 + } + }' diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests new file mode 100755 index 00000000..8c24664d --- /dev/null +++ b/scripts/run-integration-tests @@ -0,0 +1,65 @@ +#!/bin/sh + +set -eu + +test -n "${METWORX_VERSION-}" || { + printf >&2 'Metworx is required to run integration tests\n' + exit 1 +} + +test -d integration || { + printf >&2 '%s must be executed from top-level directory of bbi repo\n' "$0" + exit 1 +} + +# Note: BBI_TEST_ROOT must be under /data. +root=${BBI_TEST_ROOT-/data/bbi-tests} +mkdir -p "$root" +tdir=$(mktemp -d "$root"/run-XXXXX) +trap 'rm -rf "$tdir"' 0 + +BBI_GRID_NAME_PREFIX=$(basename "$tdir") +export BBI_GRID_NAME_PREFIX +export LOCAL=true +export MPIEXEC_PATH=/usr/bin/mpiexec +export NMQUAL=false +export NMVERSION=nm75 +export NMVERSION_NMQUAL=nm74gf +export NONMEMROOT=/opt/NONMEM +export POST_EXECUTION=true +export ROOT_EXECUTION_DIR="$tdir" +export SGE=true +export SGE_ARCH=lx-amd64 +export SGE_CELL=default +export SGE_CLUSTER_NAME=p6444 +export SGE_EXECD_PORT=6445 +export SGE_QMASTER_PORT=6444 +export SGE_ROOT=/opt/sge + +bin=$tdir/bin +mkdir "$bin" + +if test -n "${GOCOVERDIR-}" +then + printf >&2 'building binary with -cover\n' + coverarg=-cover +else + coverarg= +fi + +version=$(git describe --always --dirty) + +# shellcheck disable=SC2086 +go build $coverarg \ + -ldflags "-X github.com/metrumresearchgroup/bbi/cmd.VERSION=$version" \ + -o "$bin/bbi" cmd/bbi/main.go + +export PATH="$bin:$PATH" + +printf >&2 'bbi path: %s\nbbi version: %s\n' \ + "$(command -v bbi)" "$(bbi version)" + +cd integration +bbi init --dir /opt/NONMEM --threads 2 + +go test -count 1 "$@" ./... diff --git a/scripts/run-unit-tests b/scripts/run-unit-tests new file mode 100755 index 00000000..cea126b9 --- /dev/null +++ b/scripts/run-unit-tests @@ -0,0 +1,30 @@ +#!/bin/sh + +set -eu + +ls_pkgs () { + go list ./... | + grep -vE '/internal/valtools/[a-z]+$' | + grep -vE '/internal/tools/docgen$' | + grep -vF '/integration/' | + tr '\n' ' ' +} + +pkgs=$(ls_pkgs) +pkgs=${pkgs% *} +cpkgs=$(printf '%s' "$pkgs" | tr ' ' ',') + +run () { + go test -p 1 -count 1 "$@" +} + +if test -n "${GOCOVERDIR-}" +then + printf >&2 'testing with -cover\n' + # shellcheck disable=SC2086 + run -cover -coverpkg="$cpkgs" "$@" $pkgs \ + -args -test.gocoverdir="$GOCOVERDIR" +else + # shellcheck disable=SC2086 + run "$@" $pkgs +fi diff --git a/validation/requirements.yaml b/validation/requirements.yaml deleted file mode 100644 index df6eba61..00000000 --- a/validation/requirements.yaml +++ /dev/null @@ -1,146 +0,0 @@ -CFG-R001: - description: BBI config JSON gets created - tests: - - INT-CFG-001 -COVCOR-R001: - description: CovCor parses .cov and .cor files - tests: - - INT-COVCOR-001 -COVCOR-R002: - description: CovCor errors with bad input - tests: - - INT-COVCOR-002 -DATA-R001: - description: Has valid data path for CTL - tests: - - INT-DATA-001 -DATA-R002: - description: Fails with invalid data path - tests: - - INT-DATA-002 -DATA-R003: - description: Has valid complex path CTL And Mod - tests: - - INT-DATA-003 -INIT-R001: - description: BBI can initialize - tests: - - INT-INIT-001 - - INT-INIT-002 - - UNIT-INIT-001 - - UNIT-INIT-002 - - UNIT-INIT-003 - - UNIT-INIT-004 - - UNIT-INIT-005 -LOCAL-R001: - description: BBI completes local execution - tests: - - INT-LOCAL-001 -LOCAL-R002: - description: NMFE options end in script - tests: - - INT-LOCAL-002 -LOCAL-R003: - description: BBI can execute in parallel - tests: - - INT-LOCAL-003 -NMQ-R001: - description: NMQUAL execution succeeds - tests: - - INT-NMQ-001 -PARAM-R001: - description: Parse params for single model - tests: - - INT-PARAM-001 -PARAM-R002: - description: Parse params for directory of models - tests: - - INT-PARAM-002 -SGE-R001: - description: BBI completes SGE execution - tests: - - INT-SGE-001 -SGE-R002: - description: BBI completes parallel SGE execution - tests: - - INT-SGE-002 -SGE-R003: - description: SGE execution uses same BBI executable as the main - process by default. - tests: - - INT-INIT-001 - - UNIT-CMD-006 -SUM-R001: - description: BBI summary works with default settings - tests: - - INT-SUM-001 -SUM-R002: - description: BBI summary arguments work - tests: - - INT-SUM-002 -SUM-R003: - description: BBI summary fails with bad input - tests: - - INT-SUM-003 - - INT-SUM-006 -SUM-R004: - description: BBI summary works with no extension - tests: - - INT-SUM-004 -SUM-R005: - description: BBI summary reports elapsed times for each method - tests: - - INT-SUM-001 - - UNIT-NMP-035 -SUM-R006: - description: Correctly parse summary when OFV is NaN - tests: - - UNIT-NMP-019 - - INT-SUM-001 -SUM-R007: - description: BBI summary includes times that are zero - tests: - - INT-SUM-001 -SUM-R008: - description: Can take a list of model paths and parse summary for all models - tests: - - INT-SUM-005 -SUM-R009: - description: Should return a json array, with one top-level element for each - model being summarized, in the same order they were passed in. - tests: - - INT-SUM-005 -SUM-R010: - description: If any of the models cannot be summarized, pass the relevant - error back in the relevant position in the json array. - tests: - - INT-SUM-006 -SUM-R011: - description: Attempt to identify when a model run is incomplete or - invalid, and report an error to the caller. - tests: - - INT-SUM-003 - - INT-SUM-006 -SUM-R012: - description: BBI summary includes problem_text key in JSON output - even if lst defines $PROBLEM as an empty stirng. - tests: - - INT-SUM-001 -CMD-R001: - description: Process NMFE options - tests: - - UNIT-CMD-002 -CMD-R002: - description: Pass -maxeval=2 to NMFE by default - tests: - - INT-INIT-001 - - UNIT-CMD-002 -NMP-R001: - description: Parse Estimates From Ext - tests: - - UNIT-NMP-036 -NMP-R002: - description: Correctly parses problem text when either short or long-form - keyword is used. - tests: - - UNIT-NMP-035 diff --git a/validation/stories.yaml b/validation/stories.yaml deleted file mode 100644 index 0faf4d77..00000000 --- a/validation/stories.yaml +++ /dev/null @@ -1,112 +0,0 @@ -BBI-RUN-001: - name: Run NonMem jobs locally - description: As a user, I would like to be able to execute NonMem jobs locally, - without needing the grid, if NonMem is installed on the system. - ProductRisk: High - requirements: - - LOCAL-R001 - - LOCAL-R003 -BBI-RUN-002: - name: Run NonMem jobs on the Grid - description: As a user, I would like to be able to submit NonMem jobs to be run - on a worker node in the SGE grid. - ProductRisk: High - requirements: - - SGE-R001 - - SGE-R002 - - SGE-R003 -BBI-RUN-003: - name: Notify about issues with the data referenced in the control stream - description: As a user, I would like to be notified, and have model execution stop, - if I target a NonMem control stream with bbi, but the data file referenced therein - cannot be located. - ProductRisk: Low - requirements: - - DATA-R001 - - DATA-R002 - - DATA-R003 -BBI-RUN-004: - name: Pass NMFE options directly to NonMem - description: As a user, I would like to be able to pass some specified options, - such as license or compilation options, directly to NonMem, such that they are - expressed in the final NMFE call. - ProductRisk: Low - requirements: - - CMD-R001 - - CMD-R002 - - LOCAL-R002 -BBI-RUN-005: - name: NonMem Execution via NMQual - description: As a user, I would like to have the option to execute NonMem the same - way that autolog.pl did so that NMQual can be used. I would like an option exposed - that will trigger bbi to specify autolog.pl syntax in its executable script, rather - than the typical calls directly to NMFE. - ProductRisk: Low - requirements: - - NMQ-R001 -BBI-RUN-006: - name: Capture all configurations and write to a file - description: As a user, I would like to have all configurations used for each run - captured in files that can be stored in version control, for the sake of reproducibility. - The files should contain the merged configurations between any flags provided, - configuration files and default values to indicate exactly how the model was executed. - ProductRisk: High - requirements: - - CFG-R001 -BBI-CFG-001: - name: Initialize a project with minimum configs required for execution - description: As a user, I would like to be able to initialize a project with bbi - without building the config `bbi.yaml` file from scratch. Allowing a command to - create this file with the minimum default configuration necessary. - ProductRisk: Medium - requirements: - - INIT-R001 -BBI-SUM-001: - name: Parse model output folder - description: As a user, I want to be able to parse a summary of the files in the - NonMem output folder into either a human-readable table, or a machine-readable - structure like `.json`. - ProductRisk: Medium - requirements: - - SUM-R001 - - SUM-R002 - - SUM-R003 - - SUM-R004 - - SUM-R005 - - SUM-R006 - - SUM-R007 - - SUM-R008 - - SUM-R009 - - SUM-R010 - - SUM-R011 - - SUM-R012 - - NMP-R002 -BBI-COV-001: - name: Parse .cov and .cor files - description: As a user, I want to be able to parse the `.cov` and `.cor` files in - the NonMem output folder into a machine-readable structure like `.json`. - ProductRisk: Low - requirements: - - COVCOR-R001 - - COVCOR-R002 -BBI-SUM-002: - name: Parse batch parameter estimates - description: As a user, I would like to be able to parse the final parameter estimates - from multiple NONMEM model output directories as a batch. - ProductRisk: Low - requirements: - - PARAM-R001 - - PARAM-R002 - - NMP-R001 - - PARAM-R001 - - PARAM-R002 -BBI-SUM-003: - name: Summarize models in batch - description: As a user, I want to be able to parse a summary of NONMEM outputs - for multiple models in batch. - ProductRisk: High - requirements: - - SUM-R008 - - SUM-R009 - - SUM-R010 - - SUM-R011