support for multiple queues

[crkrenn]

@FrankD412 @jsemler

Below is a strawman spec that could support multiple queues. There would probably be a need for conductors running for each queue and communicating with each other.

-Chris

description:
    name: flyer-study
    description: |
        Running flyer.
        A simple 1D gas gun deck         

# current & future:
# batch:
#     type: slurm
#     host: rztopaz
#     bank: wbronze
#     queue: pbatch
#     gres: ignore
# recommend to add procs_per_node to default batch template
#     procs_per_node: 36

# future:
batchlist:
  - host: rztopaz
    type: slurm
    bank: wbronze
    queue: pbatch
    gres: ignore
    procs_per_node: 36
    variables:
      OUTPUT_PATH: /p/lustre1/crkrenn/studies
      - 

  - host: rztrona
    type: slurm
    bank: wbronze
    queue: pbatch
    gres: ignore
    procs_per_node: 36
    variables:
      OUTPUT_PATH: /p/lustre2/crkrenn/studies

env:
    variables:
        LOG_PATH: /usr/workspace/ntnf_sam/studies/logs
    #     OUTPUT_PATH: /p/lustre1/crkrenn/studies
    dependencies:
        paths:
            - name: ares
              path: /usr/apps/ARES/exec/ares.prod

            - name: FLYER_RUN_DRIVER
              path: /g/g21/crkrenn/ares-models/ares-models/flyer_debug/flyer.deck

            - name: FLYER
              path: /g/g21/crkrenn/ares-models/ares-models/flyer_debug

study:
    - name: run-flyer-baseline
      description: |
          A simple 1D gas gun deck      
      run:
          cmd: |
              cp -L -r $(FLYER)/* ./
              maestro_link --output-path-root $(OUTPUT_PATH)/../.. --link-directory-template {{output_path_root}}/links/{{date}}/run-{{INDEX}}/{{instance}}/{{step}} 1> maestro_link.out 2> maestro_link.err

              $(LAUNCHER) $(ares) flyer.deck -r  -def FLYt=$(FLYt) -def FLYw=$(FLYw) 
          # remove the requirement for nodes
          # nodes: 1
          # add optional 
          run_batchlist:
            - rztopaz
            - rztrona
          procs: 1
          walltime: "0:05:00"
          depends: []

global.parameters:

    FLYt:
        values: [0.09, 0.11]
        label: FLYt.%%

    FLYw:
        values: [0.09, 0.11]
        label: FLYw.%%