Buildtest Schemas

buildtest uses JSON Schema for validating buildspecs and buildtest configuration file. The json schemas are published at https://buildtesters.github.io/buildtest/ and we provide a command line interface to view schema files and examples.

Schema Naming Convention

All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.

CLI for buildtest schemas (buildtest schema)

You can use buildtest schema command to see the list of schemas supported by buildtest. The schema files are denoted by .schema.json file extension.

buildtest schema
$ buildtest schema
global.schema.json
definitions.schema.json
settings.schema.json
compiler.schema.json
spack.schema.json
script.schema.json

Shown below is the command usage of buildtest schema

buildtest schema --help
$ buildtest schema --help
usage: buildtest [options] [COMMANDS] schema [-h] [-e] [-j] [-n Schema Name]

optional arguments:
  -h, --help            show this help message and exit
  -e, --example         Show schema examples
  -j, --json            Display json schema file
  -n Schema Name, --name Schema Name
                        show schema by name (e.g., script)

You must use the --name option to select a schema, for instance if you want to view the JSON Schema for script.schema.json you can run the following:

buildtest schema --name script.schema.json --json

Schema Files

Definition Schema

This schema is used for declaring definitions that need to be reused in multiple schemas. We use $ref keyword to reference definitions from this file.

definitions.schema.json
{
  "$id": "definitions.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "JSON Schema Definitions File. ",
  "description": "This file is used for declaring definitions that are referenced from other schemas",
  "definitions": {
    "list_of_strings": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
    "string_or_list": {
      "oneOf": [
        {
          "type": "string"
        },
        {
          "$ref": "#/definitions/list_of_strings"
        }
      ]
    },
    "list_of_ints": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "integer"
      }
    },
    "int_or_list": {
      "oneOf": [
        {
          "type": "integer"
        },
        {
          "$ref": "#/definitions/list_of_ints"
        }
      ]
    },
    "regex": {
      "type": "object",
      "description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
      "required": [
        "stream",
        "exp"
      ],
      "additionalProperties": false,
      "properties": {
        "stream": {
          "type": "string",
          "enum": [
            "stdout",
            "stderr"
          ],
          "description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
        },
        "exp": {
          "type": "string",
          "description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
        }
      }
    },
    "env": {
      "type": "object",
      "description": "One or more key value pairs for an environment (key=value)",
      "minItems": 1,
      "items": {
        "type": "object",
        "minItems": 1,
        "propertyNames": {
          "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
        }
      }
    },
    "description": {
      "type": "string",
      "description": "The ``description`` field is used to document what the test is doing",
      "maxLength": 80
    },
    "summary": {
      "type": "string",
      "description": "The ``summary`` field is used to document what the test is doing and can be a multi-line string"
    },
    "tags": {
      "description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
      "$ref": "#/definitions/string_or_list"
    },
    "skip": {
      "type": "boolean",
      "description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
    },
    "executor": {
      "type": "string",
      "description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
    },
    "needs": {
      "description": "A list of test names that are dependency before runnning job",
      "type": "array",
      "items": [
        {
          "oneOf": [
            { "type": "string" },
            {
              "type": "object",
              "patternProperties": {
                "^.*$": {
                  "additionalProperties": false,
                  "properties": {
                    "state": {"$ref": "#/definitions/state"},
                    "returncode": {"$ref": "#/definitions/returncode"}
                  }
                }
              }
            }
          ]
        }
      ]
    },
    "metrics_field": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "regex": {
          "$ref": "#/definitions/regex"
        }
      }
    },
    "metrics": {
      "type": "object",
      "description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
      "patternProperties": {
        "^.*$": {
          "$ref": "#/definitions/metrics_field",
          "description": "Name of metric"
        }
      }
    },
    "state": {
      "type": "string",
      "description": "explicitly mark state of test regardless of status calculation",
      "enum": [
        "PASS",
        "FAIL"
      ]
    },
    "returncode": {
      "description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
      "$ref": "#/definitions/int_or_list"
    },
    "status": {
      "type": "object",
      "description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
      "additionalProperties": false,
      "properties": {
        "slurm_job_state": {
          "type": "string",
          "enum": [
            "COMPLETED",
            "FAILED",
            "OUT_OF_MEMORY",
            "TIMEOUT"
          ],
          "description": "This field can be used to pass test based on Slurm Job State, if there is a match buildtest will report as ``PASS`` "
        },
        "pbs_job_state":
        {
          "type": "string",
          "description": "This field can be used to pass test based on PBS Job State, if there is a match buildtest will report as ``PASS`` ",
          "enum": [
            "H",
            "S",
            "F"
          ]
        },
        "lsf_job_state": {
          "type": "string",
          "description": "This field can be used to pass test based on LSF Job State, if there is a match buildtest will report as ``PASS`` ",
          "enum": [
            "DONE",
            "EXIT"
          ]
        },

        "returncode": { "$ref": "#/definitions/returncode" },
        "regex": {
          "$ref": "#/definitions/regex",
          "description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
        },
        "runtime": {
          "type": "object",
          "description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
          "properties": {
            "min": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
            },
            "max": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
            }
          }
        },
        "state": {
          "$ref": "#/definitions/state",
          "description": "explicitly mark state of test regardless of status calculation"
        }
      }
    },
    "BB": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Create burst buffer space, this specifies #BB options in your test."
    },
    "DW": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Specify Data Warp option (#DW) when using burst buffer."
    },
    "sbatch": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #SBATCH options in test script."
    },
    "bsub": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #BSUB options in test script."
    },
    "cobalt": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #COBALT options in test script."
    },
    "pbs": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #PBS directives in test script."
    },
    "executors": {
      "type": "object",
      "description": "Define executor specific configuration",
      "patternProperties": {
        "description": "Name of executor to override configuration",
        "^.*$": {
          "additionalProperties": false,
          "properties": {
            "env": {
              "$ref": "#/definitions/env"
            },
            "vars": {
              "$ref": "#/definitions/env"
            },
            "sbatch": {
              "$ref": "#/definitions/list_of_strings"
            },
            "bsub": {
              "$ref": "#/definitions/list_of_strings"
            },
            "pbs": {
              "$ref": "#/definitions/list_of_strings"
            },
            "cobalt": {
              "$ref": "#/definitions/list_of_strings"
            },
            "BB": {
              "$ref": "#/definitions/BB"
            },
            "DW": {
              "$ref": "#/definitions/DW"
            },
            "status": {
              "$ref": "#/definitions/status"
            },
            "metrics": {
              "$ref": "#/definitions/metrics"
            }
          }
        }
      }
    },
    "cc": {
      "type": "string",
      "description": "Set C compiler wrapper"
    },
    "fc": {
      "type": "string",
      "description": "Set Fortran compiler wrapper"
    },
    "cxx": {
      "type": "string",
      "description": "Set C++ compiler wrapper"
    },
    "cflags": {
      "type": "string",
      "description": "Set C compiler flags."
    },
    "fflags": {
      "type": "string",
      "description": "Set Fortran compiler flags."
    },
    "cxxflags": {
      "type": "string",
      "description": "Set C++ compiler flags."
    },
    "ldflags": {
      "type": "string",
      "description": "Set linker flags"
    },
    "cppflags": {
      "type": "string",
      "description": "Set C or C++ preprocessor flags"
    },
    "run": {
      "type": "string",
      "description": "Specify a series of commands to run."
    },
    "module": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
             "purge": {
               "type": "boolean",
                "description": "Run ``module purge`` if purge is set"
             },
             "load": {
               "$ref": "definitions.schema.json#/definitions/list_of_strings",
               "description": "Load one or more modules via ``module load``"
             },
             "restore": {
               "description": "Load a collection name via ``module restore``",
               "type": "string"
             },
             "swap": {
               "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
               "type": "array",
               "uniqueItems": true,
               "minItems": 2,
               "maxItems": 2,
               "items": {
                 "type": "string"
               }
             }
           }
}
  }
}

Settings Schema

This schema defines how buildtest configuration file is validated.

settings.schema.json
{
  "$id": "settings.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "buildtest configuration schema",
  "type": "object",
  "required": [
    "system"
  ],
  "additionalProperties": false,
  "properties": {
    "system": {
      "type": "object",
      "patternProperties": {
        "^.*$": {
          "$ref": "#/definitions/system"
        }
      }
    }
  },
  "definitions": {
    "system": {
      "required": [
        "executors",
        "moduletool",
        "hostnames",
        "compilers"
      ],
      "additionalProperties": false,
      "type": "object",
      "properties": {
        "hostnames": {
          "type": "array",
          "description": "Specify a list of hostnames to check where buildtest can run for the given system record",
          "items": {
            "type": "string"
          }
        },
        "description": {
          "type": "string",
          "description": "system description field"
        },
        "poolsize":
        {
          "type": "integer",
          "minimum": 1,
          "description": "Specify size of Process Pool for parallel processing using ``multiprocessing.Pool``"
        },
        "buildspec_roots": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
        },
        "testdir": {
          "type": "string",
          "description": "Specify full path to test directory where buildtest will write tests."
        },
        "logdir": {
          "type": "string",
          "description": "Specify location where buildtest will write log files"
        },
        "moduletool": {
          "type": "string",
          "description": "Specify modules tool used for interacting with ``module`` command. ",
          "enum": [
            "environment-modules",
            "lmod",
            "N/A"
          ]
        },
        "timeout": {
          "type": "integer",
          "minimum": 1,
          "description": "Specify timeout duration in number of seconds"
        },
        "processor": {
          "type": "object",
          "description": "Specify processor information",
          "additionalProperties": false,
          "properties": {
            "numcpus": {"type": "integer", "minimum": 1, "description": "Specify Total Number of CPUs"},
            "sockets": {"type": "integer", "minimum": 1, "description": "Specify Number of CPU Sockets"},
            "cores": {"type": "integer", "minimum": 1, "description": "Specify Number of Physical Cores"},
            "threads_per_core": {"type": "integer", "minimum":  1, "description": "Specify Threads per Core" },
            "core_per_socket": {"type": "integer", "minimum": 1, "description": "Specify Cores per Socket"},
            "model": {"type": "string", "description": "Specify Processor Model" },
            "arch": {"type": "string", "description": "Specify processor architecture"},
            "vendor": {"type": "string", "description": "Vendor Name"}
          }
        },
        "compilers": {
          "type": "object",
          "description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
          "additionalProperties": false,
          "properties": {
            "enable_prgenv": {
              "type": "boolean",
              "description": "Enable support for Programming Environment"
            },
            "modulepath": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
            "purge": {"type": "boolean", "description": "A boolean to determine whether to purge modules via ``module purge`` when generating compiler declaration"},
            "prgenv_modules": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "gcc": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for gcc",
                  "enum": [
                    "PrgEnv-gnu"
                  ]
                },
                "intel": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for intel",
                  "enum": [
                    "PrgEnv-intel"
                  ]
                },
                "cray": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for cray",
                  "enum": [
                    "PrgEnv-cray"
                  ]
                },
                "nvhpc": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for nvhpc",
                  "enum": [
                    "PrgEnv-nvhpc",
                    "PrgEnv-nvidia"
                  ]
                }
              }
            },
            "find": {
              "type": "object",
              "additionalProperties": false,
              "description": "Find compilers by specifying regular expression that is applied to modulefile names",
              "properties": {
                "gcc": {
                  "type": "string",
                  "description": "Specify a regular expression to search for gcc compilers from your module stack"
                },
                "intel": {
                  "type": "string",
                  "description": "Specify a regular expression to search for intel compilers from your module stack"
                },
                "cray": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cray compilers from your module stack"
                },
                "clang": {
                  "type": "string",
                  "description": "Specify a regular expression to search for clang compilers from your module stack"
                },
                "cuda": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cuda compilers from your module stack"
                },
                "pgi": {
                  "type": "string",
                  "description": "Specify a regular expression to search for pgi compilers from your module stack"
                },
                "upcxx": {
                  "type": "string",
                  "description": "Specify a regular expression to search for upcxx compilers from your module stack"
                },
                "nvhpc": {
                  "type": "string",
                  "description": "Specify a regular expression to search for nvhpc compilers from your module stack"
                }
              }
            },
            "compiler": {
              "type": "object",
              "additionalProperties": false,
              "description": "Start of compiler declaration",
              "properties": {
                "gcc": {
                  "description": "Declaration of one or more GNU compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "intel": {
                  "description": "Declaration of one or more Intel compilers. ",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "cray": {
                  "description": "Declaration of one or more Cray compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "pgi": {
                  "description": "Declaration of one or more PGI compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "clang": {
                  "description": "Declaration of one or more Clang compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "cuda": {
                  "description": "Declaration of one or more CUDA compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "upcxx": {
                  "description": "Declaration of one or more UPCXX compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "nvhpc": {
                  "description": "Declaration of one or more NVHPC compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                }
              }
            }
          }
        },
        "executors": {
            "type": "object",
            "additionalProperties": false,
            "description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``, ``cobalt``. The executors are referenced in buildspec using ``executor`` field.",
            "required": ["local"],
            "properties": {
              "defaults": {
                "type": "object",
                "description": "Specify default executor settings for all executors",
                "additionalProperties": false,
                "properties": {
                  "pollinterval": {
                    "type": "integer",
                    "description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
                    "minimum": 1,
                    "default": 30
                  },
                  "maxpendtime": {
                    "$ref": "#/definitions/maxpendtime"
                  },
                  "account": {
                    "$ref": "#/definitions/account"
                  },
                  "max_jobs": {"$ref":  "#/definitions/max_jobs"}
                }
              },
              "local": {
                "type": "object",
                "description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
                "patternProperties": {
                  "^.*$": {
                    "$ref": "#/definitions/local"
                  }
                }
              },
              "lsf": {
                "type": "object",
                "description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
                "patternProperties": {
                  "^.*$": {
                    "$ref": "#/definitions/lsf"
                  }
                }
              },
              "slurm": {
                "type": "object",
                "description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
                "patternProperties": {
                  "^.*$": {
                    "$ref": "#/definitions/slurm"
                  }
                }
              },
              "cobalt": {
                "type": "object",
                "description": "The ``cobalt`` section is used for declaring Cobalt executors for running jobs using Cobalt scheduler",
                "patternProperties": {
                  "^.*$": {
                    "$ref": "#/definitions/cobalt"
                  }
                }
              },
              "pbs": {
                "type": "object",
                "description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
                "patternProperties": {
                  "^.*$": {
                    "$ref": "#/definitions/pbs"
                  }
                }
              }
            }
          },
        "cdash": {
          "type": "object",
          "description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
          "required": ["url", "project", "site"],
          "properties": {
            "url": {
              "type": "string",
              "description": "Url to CDASH server"
            },
            "project": {
              "type": "string",
              "description": "Name of CDASH project"
            },
            "site": {
              "type": "string",
              "description": "Site Name reported in CDASH"
            }
          }
        }
      }
    },
    "cc": {
      "description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
      "type": "string"
    },
    "cxx": {
      "type": "string",
      "description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "fc": {
      "type": "string",
      "description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "max_jobs": {
      "description": "Maximum number of jobs that can be run at a given time for a particular executor",
      "type": "integer",
      "minimum": 1
    },
    "compiler_section": {
      "description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
      "type": "object",
      "additionalProperties": false,
      "required": [ "cc",  "cxx",  "fc" ],
      "properties": {
        "cc": { "$ref": "#/definitions/cc" },
        "cxx": { "$ref": "#/definitions/cxx" },
        "fc": { "$ref": "#/definitions/fc" },
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "unique_string_array": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string"
      }
    },
    "disable": {"type": "boolean", "description": "Disable executor"},
    "module": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "purge": {
          "type": "boolean",
          "description": "Run ``module purge`` if purge is set"
        },
        "load": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Load one or more modules via ``module load``"
        },
        "swap": {
          "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
          "type": "array",
          "uniqueItems": true,
          "minItems": 2,
          "maxItems": 2,
          "items": { "type": "string" }
        },
         "restore": {
           "description": "Load a collection name via ``module restore``",
           "type": "string"
         }
      }
    },
    "script": {
      "type": "array",
      "additionalProperties": false,
      "items": { "type": "string" }
    },
    "maxpendtime": {
      "type": "integer",
      "description": "Cancel job if it is still pending in queue beyond maxpendtime",
      "minimum": 1,
      "default": 86400
    },
    "account": {
      "type": "string",
      "description": "Specify Job Account for charging resources"
    },
    "local": {
      "type": "object",
      "description": "An instance object of local executor",
      "additionalProperties": false,
      "required": [ "shell" ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "shell": {
          "type": "string",
          "description": "Specify the shell launcher you want to use when running tests locally"
        },
        "before_script": { "#ref": "#/definitions/script" },
        "max_jobs": {"$ref":  "#/definitions/max_jobs"},
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "slurm": {
      "type": "object",
      "additionalProperties": false,
      "description": "An instance object of slurm executor",
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [ "sbatch" ],
          "description": "Specify the slurm batch scheduler to use. This overrides the default ``launcher`` field. This must be ``sbatch``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
        },
        "cluster": {
          "type": "string",
          "description": "Specify the slurm cluster you want to use ``-M <cluster>``"
        },
        "partition": {
          "type": "string",
          "description": "Specify the slurm partition you want to use ``-p <partition>``"
        },
        "qos": {
          "type": "string",
          "description": "Specify the slurm qos you want to use ``-q <qos>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "max_jobs": {"$ref":  "#/definitions/max_jobs"},
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "lsf": {
      "type": "object",
      "description": "An instance object of lsf executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [ "bsub" ],
          "description": "Specify the lsf batch scheduler to use. This overrides the default ``launcher`` field. It must be ``bsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "max_jobs": {"$ref":  "#/definitions/max_jobs"},
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "cobalt": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [ "qsub" ],
          "description": "Specify the cobalt batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "max_jobs": {"$ref":  "#/definitions/max_jobs"},
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "pbs": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [ "qsub" ],
          "description": "Specify the pbs batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "max_jobs": {"$ref":  "#/definitions/max_jobs"},
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    }
  }
}

Global Schema

This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema

global.schema.json
{
  "$id": "global.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "global schema",
  "description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
  "type": "object",
  "required": ["buildspecs"],
  "additionalProperties": false,
  "properties": {
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "maintainers": {
      "type": "array",
      "description": "One or more maintainers or aliases",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
   "buildspecs": {
     "type": "object",
     "description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
     "propertyNames": {
       "pattern": "^[A-Za-z_.-][A-Za-z0-9_.-]*$",
       "maxLength": 48
     }
   }
  }
}

Script Schema

This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when type: script is specified. For more details on script schema see Script Schema.

script.schema.json
{
  "$id": "script.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "script schema version",
  "description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
  "type": "object",
  "required": [
    "type",
    "run",
    "executor"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^script$",
      "description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "summary": {
      "$ref": "definitions.schema.json#/definitions/summary"
    },
    "sbatch": {
      "$ref": "definitions.schema.json#/definitions/sbatch"
    },
    "bsub": {
      "$ref": "definitions.schema.json#/definitions/bsub"
    },
    "cobalt": {
      "$ref": "definitions.schema.json#/definitions/cobalt"
    },
    "pbs": {
      "$ref": "definitions.schema.json#/definitions/pbs"
    },
    "BB": {
      "$ref": "definitions.schema.json#/definitions/BB"
    },
    "DW": {
      "$ref": "definitions.schema.json#/definitions/DW"
    },
    "env": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "vars": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "needs": {
      "$ref": "definitions.schema.json#/definitions/needs"
    },
    "shell": {
      "type": "string",
      "description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed"
    },
    "shebang": {
      "type": "string",
      "description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
    },
    "run": {
      "$ref": "definitions.schema.json#/definitions/run"
    },
    "status": {
      "$ref": "definitions.schema.json#/definitions/status"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    },
    "executors": {
      "$ref": "definitions.schema.json#/definitions/executors"
    },
    "compilers": {
      "type": "object",
      "required": [
        "name"
      ],
      "additionalProperties": false,
      "properties": {
        "name": {
          "description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "exclude": {
          "description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "default": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "gcc": {
              "$ref": "#definitions/default_compiler_config"
            },
            "intel": {
              "$ref": "#definitions/default_compiler_config"
            },
            "pgi": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cray": {
              "$ref": "#definitions/default_compiler_config"
            },
            "clang": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cuda": {
              "$ref": "#definitions/default_compiler_config"
            },
            "upcxx": {
              "$ref": "#definitions/default_compiler_config"
            },
            "nvhpc": {
              "$ref": "#definitions/default_compiler_config"
            }
          }
        },
        "config": {
          "type": "object",
          "description": "Specify compiler configuration based on named compilers.",
          "patternProperties": {
            "^.*$": {
              "$ref": "#definitions/compiler_declaration"
            }
          }
        }
      }
    }
  },
  "definitions": {
    "compiler_declaration": {
      "type": "object",
      "description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
      "additionalProperties": false,
      "properties": {
        "cc": {
          "$ref": "definitions.schema.json#/definitions/cc"
        },
        "fc": {
          "$ref": "definitions.schema.json#/definitions/fc"
        },
        "cxx": {
          "$ref": "definitions.schema.json#/definitions/cxx"
        },
        "cflags": {
          "$ref": "definitions.schema.json#/definitions/cflags"
        },
        "fflags": {
          "$ref": "definitions.schema.json#/definitions/fflags"
        },
        "cxxflags": {
          "$ref": "definitions.schema.json#/definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "definitions.schema.json#/definitions/ldflags"
        },
        "cppflags": {
          "$ref": "definitions.schema.json#/definitions/cppflags"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "run": {
          "$ref": "definitions.schema.json#/definitions/run"
        },
        "module": {
          "$ref": "definitions.schema.json#/definitions/module"
        }
      },
      "default_compiler_config": {
        "type": "object",
        "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
        "properties": {
          "cc": {
            "$ref": "definitions.schema.json#/definitions/cc"
          },
          "fc": {
            "$ref": "definitions.schema.json#/definitions/fc"
          },
          "cxx": {
            "$ref": "definitions.schema.json#/definitions/cxx"
          },
          "cflags": {
            "$ref": "definitions.schema.json#/definitions/cflags"
          },
          "fflags": {
            "$ref": "definitions.schema.json#/definitions/fflags"
          },
          "cxxflags": {
            "$ref": "definitions.schema.json#/definitions/cxxflags"
          },
          "ldflags": {
            "$ref": "definitions.schema.json#/definitions/ldflags"
          },
          "cppflags": {
            "$ref": "definitions.schema.json#/definitions/cppflags"
          },
          "env": {
            "$ref": "definitions.schema.json#/definitions/env"
          },
          "vars": {
            "$ref": "definitions.schema.json#/definitions/env"
          },
          "run": {
            "$ref": "definitions.schema.json#/definitions/run"
          }
        }
      }
    },
    "default_compiler_config": {
      "type": "object",
      "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
      "properties": {
        "cc": {
          "$ref": "definitions.schema.json#/definitions/cc"
        },
        "fc": {
          "$ref": "definitions.schema.json#/definitions/fc"
        },
        "cxx": {
          "$ref": "definitions.schema.json#/definitions/cxx"
        },
        "cflags": {
          "$ref": "definitions.schema.json#/definitions/cflags"
        },
        "fflags": {
          "$ref": "definitions.schema.json#/definitions/fflags"
        },
        "cxxflags": {
          "$ref": "definitions.schema.json#/definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "definitions.schema.json#/definitions/ldflags"
        },
        "cppflags": {
          "$ref": "definitions.schema.json#/definitions/cppflags"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "run": {
          "$ref": "definitions.schema.json#/definitions/run"
        }
      }
    }
  }
}

Compiler Schema

This is the compiler schema used for validating buildspecs that define test using type: compiler. This schema is used for compiling a single source code. For more details see Using Compiler Schema

compiler.schema.json
{
  "$id": "compiler.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "compiler schema",
  "description": "The compiler schema is of ``type: compiler`` in sub-schema which is used for compiling and running programs",
  "type": "object",
  "required": [
    "type",
    "source",
    "compilers",
    "executor"
  ],
  "definitions": {
    "cc": {
      "type": "string",
      "description": "Set C compiler wrapper"
    },
    "fc": {
      "type": "string",
      "description": "Set Fortran compiler wrapper"
    },
    "cxx": {
      "type": "string",
      "description": "Set C++ compiler wrapper"
    },
    "cflags": {
      "type": "string",
      "description": "Set C compiler flags."
    },
    "fflags": {
      "type": "string",
      "description": "Set Fortran compiler flags."
    },
    "cxxflags": {
      "type": "string",
      "description": "Set C++ compiler flags."
    },
    "ldflags": {
      "type": "string",
      "description": "Set linker flags"
    },
    "cppflags": {
      "type": "string",
      "description": "Set C or C++ preprocessor flags"
    },
    "pre_build": {
      "type": "string",
      "description": "Run commands before building program"
    },
    "post_build": {
      "type": "string",
      "description": "Run commands after building program"
    },
    "pre_run": {
      "type": "string",
      "description": "Run commands before running program"
    },
    "post_run": {
      "type": "string",
      "description": "Run commands after running program"
    },
    "run": {
      "type": "string",
      "description": "Run command for launching compiled binary"
    },
    "default_compiler_all": {
      "type": "object",
      "description": "Specify compiler configuration for all compiler groups. Use the ``all`` property if configuration is shared across compiler groups. This property can be overridden in compiler group or named compiler in ``config`` section.",
      "additionalProperties": false,
      "properties": {
        "sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
        "bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
        "cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
        "pbs": { "$ref":  "definitions.schema.json#/definitions/pbs" },
        "BB": { "$ref": "definitions.schema.json#/definitions/BB" },
        "DW": { "$ref": "definitions.schema.json#/definitions/DW" },
        "env": { "$ref": "definitions.schema.json#/definitions/env" },
        "vars": { "$ref": "definitions.schema.json#/definitions/env" },
        "status": { "$ref": "definitions.schema.json#/definitions/status" },
        "pre_build": { "$ref": "#definitions/pre_build" },
        "post_build": { "$ref": "#definitions/post_build" },
        "pre_run": { "$ref": "#definitions/pre_run" },
        "post_run": { "$ref": "#definitions/post_run" },
        "run": { "$ref": "#definitions/run" }
      }
    },
    "default_compiler_config": {
      "type": "object",
      "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
      "properties": {
        "cc": { "$ref": "#definitions/cc" },
        "fc": { "$ref": "#definitions/fc" },
        "cxx": { "$ref": "#definitions/cxx" },
        "cflags": { "$ref": "#definitions/cflags" },
        "fflags": { "$ref": "#definitions/fflags" },
        "cxxflags": { "$ref": "#definitions/cxxflags" },
        "ldflags": { "$ref": "#definitions/ldflags" },
        "cppflags": { "$ref": "#definitions/cppflags" },
        "sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
        "bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
        "cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
        "pbs": { "$ref":  "definitions.schema.json#/definitions/pbs"},
        "BB": { "$ref": "definitions.schema.json#/definitions/BB"},
        "DW": { "$ref": "definitions.schema.json#/definitions/DW"},
        "env": { "$ref": "definitions.schema.json#/definitions/env" },
        "vars": { "$ref": "definitions.schema.json#/definitions/env" },
        "status": { "$ref": "definitions.schema.json#/definitions/status" },
        "pre_build": { "$ref": "#definitions/pre_build" },
        "post_build": { "$ref": "#definitions/post_build" },
        "pre_run": { "$ref": "#definitions/pre_run" },
        "post_run": { "$ref": "#definitions/post_run" },
        "run": { "$ref": "#definitions/run" }
      }
    },
    "compiler_declaration": {
      "type": "object",
      "description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
      "additionalProperties": false,
      "properties": {
        "cc": { "$ref": "#definitions/cc" },
        "fc": { "$ref": "#definitions/fc" },
        "cxx": { "$ref": "#definitions/cxx" },
        "cflags": { "$ref": "#definitions/cflags" },
        "fflags": { "$ref": "#definitions/fflags" },
        "cxxflags": { "$ref": "#definitions/cxxflags" },
        "ldflags": { "$ref": "#definitions/ldflags" },
        "cppflags": { "$ref": "#definitions/cppflags" },
        "sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
        "bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
        "cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
        "pbs": { "$ref":  "definitions.schema.json#/definitions/pbs"},
        "BB": { "$ref": "definitions.schema.json#/definitions/BB"},
        "DW": { "$ref": "definitions.schema.json#/definitions/DW"},
        "env": { "$ref": "definitions.schema.json#/definitions/env" },
        "vars": { "$ref": "definitions.schema.json#/definitions/env" },
        "status": { "$ref": "definitions.schema.json#/definitions/status" },
        "pre_build": { "$ref": "#definitions/pre_build" },
        "post_build": { "$ref": "#definitions/post_build" },
        "pre_run": { "$ref": "#definitions/pre_run" },
        "post_run": { "$ref": "#definitions/post_run" },
        "run": { "$ref": "#definitions/run" },
        "module": { "$ref": "definitions.schema.json#/definitions/module" }
      }
    }
  },
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^compiler$",
      "description": "Select schema type to use when validating buildspec. This must be of set to ``compiler``."
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "summary": {
      "$ref": "definitions.schema.json#/definitions/summary"
    },
    "needs": {
      "$ref": "definitions.schema.json#/definitions/needs"
    },
    "compilers": {
      "type": "object",
      "required": [
        "name"
      ],
      "additionalProperties": false,
      "properties": {
        "name": {
          "description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "exclude": {
          "description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "default": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "all": {
              "$ref": "#definitions/default_compiler_all"
            },
            "gcc": {
              "$ref": "#definitions/default_compiler_config"
            },
            "intel": {
              "$ref": "#definitions/default_compiler_config"
            },
            "pgi": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cray": {
              "$ref": "#definitions/default_compiler_config"
            },
            "clang": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cuda": {
              "$ref": "#definitions/default_compiler_config"
            },
            "upcxx": {
              "$ref": "#definitions/default_compiler_config"
            },
            "nvhpc": {
              "$ref": "#definitions/default_compiler_config"
            }
          }
        },
        "config": {
          "type": "object",
          "description": "Specify compiler configuration based on named compilers.",
          "patternProperties": {
            "^.*$": {
              "$ref": "#definitions/compiler_declaration"
            }
          }
        }
      }
    },
    "source": {
      "type": "string",
      "description": "Specify a source file for compilation, the file can be relative path to buildspec or an absolute path"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    }
  }
}

Spack Schema

This schema is used for writing tests with spack package manager using type: spack field. For more details see Buildtest Spack Integration.

spack.schema.json
{
  "$id": "spack.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "spack schema version",
  "description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
  "type": "object",
  "required": [
    "type",
    "executor",
    "spack"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^spack$",
      "description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
    },
    "description": { "$ref": "definitions.schema.json#/definitions/description" },
    "summary": { "$ref": "definitions.schema.json#/definitions/summary" },
    "executor": { "$ref": "definitions.schema.json#/definitions/executor" },
    "env": { "$ref": "definitions.schema.json#/definitions/env" },
    "vars": { "$ref": "definitions.schema.json#/definitions/env" },
    "sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
    "bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
    "cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
    "pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
    "BB": { "$ref": "definitions.schema.json#/definitions/BB" },
    "DW": { "$ref": "definitions.schema.json#/definitions/DW" },
    "skip": { "$ref": "definitions.schema.json#/definitions/skip" },
    "tags": { "$ref": "definitions.schema.json#/definitions/tags" },
    "status": { "$ref": "definitions.schema.json#/definitions/status" },
    "metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
    "executors": { "$ref": "definitions.schema.json#/definitions/executors" },
    "pre_cmds": {
      "type": "string",
      "description": "Shell commands run before spack"
    },
    "post_cmds": {
      "type": "string",
      "description": "Shell commands run after spack"
    },
    "needs": {
      "$ref": "definitions.schema.json#/definitions/needs"
    },
    "spack": {
      "type": "object",
      "description": "Entry point to spack configuration",
      "additionalProperties": false,
      "properties": {
        "root": {
          "type": "string",
          "description": "Specify location for root of spack directory"
        },
        "compiler_find": {
          "type": "boolean",
          "description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror by running ``spack mirror add``"
        },
        "env": {
          "$ref": "#definitions/env",
          "description": "Manage spack environments via ``spack env`` command"
        },
        "install": {
          "$ref": "#definitions/install",
          "description": "Install spack packages by running ``spack install``. "
        },
        "verify_spack": {
          "type": "boolean",
          "description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
          "default": true
        },
        "test": {
          "$ref": "#definitions/test",
          "description": "Entry point to ``spack test``"
        }
      }
    }
  },
  "definitions": {
    "env": {
      "additionalProperties": false,
      "type": "object",
      "description": "Used for managing spack environment using ``spack env`` command. ",
      "properties": {
        "create": {
          "additionalProperties": false,
          "description": "Create a spack environment via ``spack env create``",
          "type": "object",
          "properties": {
            "remove_environment": {
              "type": "boolean",
              "description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
              "default": false
            },
            "name": {
              "type": "string",
              "description": "Name of spack environment to create"
            },
            "manifest": {
              "type": "string",
              "description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env create`` command"
            },
            "dir": {
              "type": "string",
              "description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
            }
          }
        },
        "activate": {
          "additionalProperties": false,
          "type": "object",
          "description": "Activate a spack environment via ``spack env activate``",
          "properties": {
            "name": {
              "type": "string",
              "description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env activate`` command"
            },
            "dir": {
              "type": "string",
              "description": "Activate spack environment from directory."
            }
          }
        },
        "rm": {
          "additionalProperties": false,
          "description": "Remove an existing spack environment via ``spack env rm``.",
          "type": "object",
          "required": [
            "name"
          ],
          "properties": {
            "name": {
              "type": "string",
              "description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
            }
          }
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror in spack environment by running ``spack mirror add``"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
        },
        "concretize": {
          "type": "boolean",
          "description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
        }
      }
    },
    "install": {
      "description": "Install spack packages using ``spack install`` command",

      "type": "object",
      "properties": {
        "options": {
          "type": "string",
          "description": "Pass options to ``spack install`` command"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "List of specs to install using ``spack install`` command"
        }
      }
    },
    "test": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "run",
        "results"
      ],
      "properties": {
        "remove_tests": {
          "type": "boolean",
          "description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
        },
        "run": {
          "description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
          "type": "object",
          "required": [
            "specs"
          ],
          "additionalProperties": false,
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test run``"
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "List of specs to run tests by running ``spack test run <specs>``."
            }
          }
        },
        "results": {
          "type": "object",
          "description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
          "additionalProperties": false,
          "anyOf": [
                    {"required": ["specs"] },
                    {"required": ["suite"] },
                    {"required": ["specs","suite"] }
          ],
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test results``"
            },
            "suite": {
                  "$ref": "definitions.schema.json#/definitions/list_of_strings",
                  "description": "Report results by  suite name by running ``spack test results <suite>``."
            },
            "specs": {
                  "$ref": "definitions.schema.json#/definitions/list_of_strings",
                  "description": "Report result by spec name by running ``spack test run -- <specs>``."
            }
          }
        }
      }
    }
  }
}

Schema Examples

The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.

In order to view an example you can run:

buildtest schema -n <schema> --example

Settings Schema Examples

buildtest schema -n settings.schema.json --example
$ buildtest schema -n settings.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    executors:                                                                  
      defaults:                                                                 
        pollinterval: 10                                                        
        maxpendtime: 45                                                         
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      lsf:                                                                      
        batch:                                                                  
          description: "LSF Executor name 'batch' that submits jobs to 'batch' q
          queue: batch                                                          
          account: developer                                                    
          options: ["-W 20"]                                                    
          before_script: |                                                      
            time                                                                
            echo "commands run before job"                                      
        test:                                                                   
          description: "LSF Executor name 'test' that submits jobs to 'test' que
          launcher: bsub                                                        
          queue: test                                                           
          account: qa                                                           
          options: ["-W 20"]                                                    
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    buildspec_roots:                                                            
      - $HOME/buildtest-cori                                                    
    testdir: /tmp/buildtest                                                     
    executors:                                                                  
      defaults:                                                                 
        pollinterval: 20                                                        
        maxpendtime: 30                                                         
        account: admin                                                          
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      slurm:                                                                    
        normal:                                                                 
          options: ["-C haswell"]                                               
          qos: normal                                                           
          before_script: |                                                      
            time                                                                
            echo "commands run before job"                                      
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: N/A                                                             
    poolsize: 1                                                                 
    executors:                                                                  
      defaults:                                                                 
         pollinterval: 10                                                       
         maxpendtime: 30                                                        
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      pbs:                                                                      
        workq:                                                                  
          queue: workq                                                          
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: N/A                                                             
    poolsize: 1                                                                 
    executors:                                                                  
      local:                                                                    
        bash:                                                                   
          description: submit jobs on local machine                             
          shell: bash -v                                                        
      slurm:                                                                    
        haswell:                                                                
          launcher: sbatch                                                      
          options: ["-p haswell", "-t 00:10"]                                   
      lsf:                                                                      
        batch:                                                                  
          launcher: bsub                                                        
          queue: batch                                                          
          options: ["-q batch", "-t 00:10"]                                     
      cobalt:                                                                   
        normal:                                                                 
          launcher: qsub                                                        
          queue: normal                                                         
          options: ["-n 1", "-t 10"]                                            
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    #logdir: $BUILDTEST_ROOT/logs                                               
    #testdir: $BUILDTEST_ROOT/tests                                             
    moduletool: N/A                                                             
    poolsize: 1                                                                 
    cdash:                                                                      
      url: https://my.cdash.org                                                 
      project: buildtest                                                        
      site: laptop                                                              
    processor:                                                                  
      numcpus: 8                                                                
      cores: 4                                                                  
      threads_per_core: 2                                                       
      sockets: 1                                                                
      model: "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz"                         
    executors:                                                                  
      local:                                                                    
        bash:                                                                   
          description: submit jobs on local machine using bash shell            
          shell: bash                                                           
          module:                                                               
            purge: True                                                         
            restore: "Default"                                                  
            load: ["gcc"]                                                       
        sh:                                                                     
          description: submit jobs on local machine using sh shell              
          shell: sh                                                             
          before_script: |                                                      
            date                                                                
            echo "these commands will be run"                                   
        csh:                                                                    
          description: submit jobs on local machine using csh shell             
          shell: csh -x                                                         
        tcsh:                                                                   
          description: submit jobs on local machine using tcsh shell            
          shell: /bin/tcsh                                                      
        zsh:                                                                    
          description: submit jobs on local machine using zsh shell             
          shell: /bin/zsh                                                       
                                                                                
    compilers:                                                                  
      find:                                                                     
        gcc: "^(gcc|GCC|PrgEnv-gnu)"                                            
        intel: "^(intel|Intel|PrgEnv-intel)"                                    
        cray: "^(cray|PrgEnv-cray)"                                             
        clang: "^(clang|Clang)"                                                 
        cuda: "^(cuda|CUDA)"                                                    
        pgi: "^(pgi|PGI|PrgEnv-pgi)"                                            
                                                                                
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
          gcc@7.2.0:                                                            
            cc: 'cc'                                                            
            cxx: 'cxx'                                                          
            fc: 'fc'                                                            
            module:                                                             
              load:                                                             
              - gcc/7.2.0                                                       
        intel:                                                                  
          intel@2019:                                                           
            cc: 'icc'                                                           
            cxx: 'icpc'                                                         
            fc: 'ifort'                                                         
            module:                                                             
              purge: True                                                       
              load:                                                             
              - gcc/7.2.0                                                       
              - intel/2019                                                      
        cray:                                                                   
          craype@2.6.2:                                                         
            cc: 'cc'                                                            
            cxx: 'CC'                                                           
            fc: 'fc'                                                            
            module:                                                             
              load: [craype/2.6.2]                                              
              swap: [PrgEnv-gnu, PrgEnv-cray]                                   
                                                                                
        clang:                                                                  
          clang@12.0.0:                                                         
            cc: 'clang'                                                         
            cxx: 'clang++'                                                      
            fc: 'None'                                                          
            module:                                                             
              load: [clang/12.0]                                                
        cuda:                                                                   
          cuda@11.0:                                                            
            cc: 'nvcc'                                                          
            cxx: 'nvcc'                                                         
            fc: 'None'                                                          
            module:                                                             
              load: [cuda/11.0]                                                 
        pgi:                                                                    
          pgi@18.0:                                                             
            cc: 'pgcc'                                                          
            cxx: 'pgc++'                                                        
            fc: 'pgfortran'                                                     
            module:                                                             
              load: [pgi/18.0]                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    executors:                                                                  
      defaults:                                                                 
         maxpendtime: 30                                                        
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      cobalt:                                                                   
        knl:                                                                    
          queue: knl                                                            
                                                                                
        haswell:                                                                
          queue: haswell                                                        
                                                                                
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran

Global Schema Examples

buildtest schema -n global.schema.json --example
$ buildtest schema -n global.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  hostname:                                                                     
    type: script                                                                
    run: "hostname"                                                             
maintainers: [shahzebsiddiqui, shahzebsiddiqui]                                 
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
# wrong type for maintainers key, expects a string                              
maintainers: 1                                                                  
buildspecs:                                                                     
  hostname:                                                                     
    type: script                                                                
    run: "hostname"                                                             
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
# this test fails because it exceeds 32 character length for test name          
buildspecs:                                                                     
  _________this_test_exceeds_character_length______________:                    
    type: script                                                                
    run: hostname                                                               
    executor: generic.local.bash                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  # invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$"
  (badname:                                                                     
    type: script                                                                
    run: "ping login 1"                                                         
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  # testing all caps                                                            
  ABCDEFGHIJKLMNOPQRSTUVWXYZ:                                                   
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing all lowercase letters                                               
  abcdefghijklmnopqrstuvwxyz:                                                   
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' in beginning followed by all numbers                            
  _0123456789:                                                                  
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' in combination with caps, lowercase and numbers                 
  _ABCDEFabcdef0123456789:                                                      
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' at end of word                                                  
  abcdefghijklmnopqrstuvwxyz_:                                                  
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '.' in beginning of word                                            
  .helloworld:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '.' in middle of word                                               
  hello.world:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '.' at end of word                                                  
  helloworld.:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '-' in middle of word                                               
  hello-world:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '-' at end of word                                                  
  helloworld-:                                                                  
    type: script                                                                
    run: hostname

Script Schema Examples

buildtest schema -n script.schema.json --example
$ buildtest schema -n script.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  invalid_test_name_&!@#$%:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: "invalid test name"                                            
                                                                                
  invalid_bash:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    shell: "bash-missing-run"                                                   
                                                                                
  missing_run_key:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid key name roon, missing run key                         
    roon: |                                                                     
        systemctl is-active slurmd                                              
        systemctl is-enabled slurmd | grep enabled                              
                                                                                
  invalid_env_type:                                                             
    type: script                                                                
    executor: generic.local.bash                                                
    description: env key should be a dictionary                                 
    env:                                                                        
      - FOO=BAR                                                                 
    run: echo $FOO                                                              
                                                                                
  invalid_vars_type:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: var key should be a dictionary                                 
    vars:                                                                       
      - FOO=BAR                                                                 
    run: echo $FOO                                                              
                                                                                
                                                                                
  invalid_description:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description:                                                                
      - "Multi Line description"                                                
      - "is not accepted"                                                       
                                                                                
                                                                                
  invalid_summary:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: summary field must be a string                                 
    summary: 1                                                                  
                                                                                
                                                                                
  invalid_regex_stream:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of invalid regex stream                
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: file                                                            
        exp: "world$"                                                           
                                                                                
                                                                                
  regex_additionalProperties_test:                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: Testing for additional properties in regex field               
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: stdout                                                          
        exp: "world$"                                                           
        X: 1                                                                    
                                                                                
  missing_regex_exp:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of missing key 'exp' in regex          
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: stdout                                                          
                                                                                
  invalid_returncode_type:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of invalid return code type            
    run: hostname                                                               
    status:                                                                     
      returncode: ["1"]                                                         
                                                                                
  empty_returncode_list:                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: An empty returncode list will cause an error                   
    run: hostname                                                               
    status:                                                                     
      returncode: []                                                            
                                                                                
  non_int_returncodes:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: The returncode must be an int and not a number                 
    run: exit 1                                                                 
    status:                                                                     
      returncode:  1.01                                                         
                                                                                
  non_int_returncodes_list:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: The returncode must be a list of integers and no numbers       
    run: exit 1                                                                 
    status:                                                                     
      returncode:  [1, 2.230]                                                   
                                                                                
  invalid_shell_type:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid shell type must be a string                            
    shell: ["/bin/bash"]                                                        
    run: hostname                                                               
                                                                                
  invalid_type_shell_shebang:                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid type for shell shebang, must be a string               
    shebang: ["#!/bin/bash"]                                                    
    run: hostname                                                               
                                                                                
  invalid_skip_value:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid value for skip, must be boolean                        
    skip: 1                                                                     
    run: hostname                                                               
                                                                                
  empty_tags:                                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: tag list can't be empty, requires one item.                    
    tags: []                                                                    
    run: hostname                                                               
                                                                                
  non_unique_tags:                                                              
    type: script                                                                
    executor: local.bash                                                        
    description: tag names must be unique                                       
    tags: ["network", "network"]                                                
    run: hostname                                                               
                                                                                
  invalid_tags_value:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid tag value must be all string items                     
    tags: ["network", 400 ]                                                     
    run: hostname                                                               
                                                                                
  additionalProperties_test:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: additional properties are not allowed so any invalid key/value 
    FOO: BAR                                                                    
    run: hostname                                                               
                                                                                
  additionalProperties_status:                                                  
    type: script                                                                
    executor: generic.local.bash                                                
    description: test additional properties in status object. This is not allowe
    sbatch: [ "-n 2", "-q normal", "-t 10"]                                     
    run: hostname                                                               
    status:                                                                     
      slurm_job_state: "COMPLETED"                                              
      FOO: BAR                                                                  
                                                                                
  invalid_runtime_min:                                                          
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Invalid type for min property in runtime. Must be an integer o
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: "1"                                                                
                                                                                
  runtime_min_must_exceed_0:                                                    
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "The runtime must exceed 0"                                    
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: -1                                                                 
                                                                                
  invalid_slurm_job_state:                                                      
    type: script                                                                
    executor: generic.local.sh                                                  
    description: invalid value for slurm_job_state, should raise error with enum
    sbatch:                                                                     
      - "-n 2"                                                                  
      - "-q normal"                                                             
      - "-t 10"                                                                 
    run: hostname                                                               
    status:                                                                     
      slurm_job_state: "FINISH"                                                 
                                                                                
  invalid_metrics_additional_property:                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for additional property for metrics property              
    vars:                                                                       
      FOO: BAR                                                                  
    run: echo $FOO                                                              
    metrics:                                                                    
      foo:                                                                      
        variable: FOO                                                           
                                                                                
  invalid_metrics_type:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: metrics property is an object, testing for type                
    vars:                                                                       
      FOO: BAR                                                                  
    run: echo $FOO                                                              
    metrics: FOO                                                                
                                                                                
  executors_invalid_var_type:                                                   
    type: script                                                                
    executor: "generic.local.(bash|sh|zsh)"                                     
    description: Invalid type field for 'vars'                                  
    tags: [tutorials]                                                           
    run: echo $FOO                                                              
    executors:                                                                  
      generic.local.bash:                                                       
        vars: ["FOO=BAR"]                                                       
                                                                                
  executors_additionalProperties:                                               
    type: script                                                                
    executor: "generic.local.(bash|sh|zsh)"                                     
    description: Testing for additional properties in 'executors'               
    tags: [tutorials]                                                           
    run: hostname                                                               
    sbatch: ["-N 4"]                                                            
    executors:                                                                  
      generic.local.bash:                                                       
        sbatch: ["-n 4", "-N 1", "-t 30"]                                       
        FOO: BAR                                                                
      generic.local.sh:                                                         
        sbatch: ["-n 8", "-N 1", "-t 60"]                                       
      generic.local.zsh:                                                        
        sbatch: ["-n 16", "-N 2", "-t 120"]                                     
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  multiline_run:                                                                
    executor: generic.local.bash                                                
    type: script                                                                
    description: multiline run command                                          
    run: |                                                                      
      echo "1"                                                                  
      echo "2"                                                                  
                                                                                
  single_command_run:                                                           
    executor: generic.local.bash                                                
    type: script                                                                
    description: single command as a string for run command                     
    run: "hostname"                                                             
                                                                                
  declare_env:                                                                  
    executor: generic.local.bash                                                
    type: script                                                                
    description: declaring environment variables                                
    env:                                                                        
      FOO: BAR                                                                  
      X: 1                                                                      
    run: |                                                                      
      echo $FOO                                                                 
      echo $X                                                                   
                                                                                
  declare_vars:                                                                 
    executor: generic.local.bash                                                
    type: script                                                                
    description: declaring variables                                            
    vars:                                                                       
      First: Bob                                                                
      Last:  Bill                                                               
    run: |                                                                      
      echo "First:" $First                                                      
      echo "Last:" $Last                                                        
                                                                                
                                                                                
  declare_shell_sh:                                                             
    executor: generic.local.sh                                                  
    type: script                                                                
    description: declare shell name to sh                                       
    shell: sh                                                                   
    run: hostname                                                               
                                                                                
  declare_shell_bash:                                                           
    executor: generic.local.bash                                                
    type: script                                                                
    description: declare shell name to bash                                     
    shell: bash                                                                 
    run: hostname                                                               
                                                                                
  declare_shell_python:                                                         
    executor: generic.local.bash                                                
    type: script                                                                
    description: declare shell name to python                                   
    shell: python                                                               
    run: |                                                                      
      print("Hello World")                                                      
                                                                                
  declare_shell_bin_bash:                                                       
    executor: generic.local.bash                                                
    type: script                                                                
    description: declare shell name to /bin/bash                                
    shell: "/bin/bash -e"                                                       
    run: hostname                                                               
                                                                                
  declare_shell_name_bin_sh:                                                    
    executor: generic.local.sh                                                  
    type: script                                                                
    description: declare shell name to /bin/sh                                  
    shell: "/bin/sh -e"                                                         
    run: hostname                                                               
                                                                                
  declare_shell_opts:                                                           
    executor: generic.local.sh                                                  
    type: script                                                                
    description: declare shell name to sh                                       
    shell: "sh -e"                                                              
    run: hostname                                                               
                                                                                
  declare_shell_bin_zsh:                                                        
    executor: generic.local.zsh                                                 
    type: script                                                                
    description: declare shell zsh                                              
    shell: "zsh"                                                                
    run: hostname                                                               
                                                                                
  declare_shell_zsh:                                                            
    executor: generic.local.zsh                                                 
    type: script                                                                
    description: declare shell /bin/zsh                                         
    shell: "zsh"                                                                
    run: hostname                                                               
                                                                                
  declare_shell_bin_csh:                                                        
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/csh                                         
    shell: "/bin/csh"                                                           
    run: hostname                                                               
                                                                                
  declare_shell_csh:                                                            
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/tcsh                                        
    shell: "csh"                                                                
    run: hostname                                                               
                                                                                
  declare_shell_bin_tcsh:                                                       
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/tcsh                                        
    shell: "/bin/tcsh"                                                          
    run: hostname                                                               
                                                                                
                                                                                
  declare_shell_tcsh:                                                           
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell tcsh                                             
    shell: "tcsh"                                                               
    run: hostname                                                               
                                                                                
  declare_shebang:                                                              
    executor: generic.local.bash                                                
    type: script                                                                
    description: declare shell name to sh                                       
    shebang: "#!/usr/bin/env bash"                                              
    run: hostname                                                               
                                                                                
  status_returncode_list:                                                       
    executor: generic.local.bash                                                
    type: script                                                                
    description: The returncode can be a list of integers                       
    run: exit 0                                                                 
    status:                                                                     
      returncode: [0]                                                           
                                                                                
  status_returncode_int:                                                        
    executor: generic.local.bash                                                
    type: script                                                                
    description: The returncode can be an integer to match with single returncod
    run: exit 0                                                                 
    status:                                                                     
      returncode: 0                                                             
                                                                                
  status_regex:                                                                 
    executor: generic.local.bash                                                
    type: script                                                                
    description: This test pass with a regular expression status check          
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: stdout                                                          
        exp: "^$"                                                               
                                                                                
  status_regex_returncode:                                                      
    executor: generic.local.bash                                                
    type: script                                                                
    description: This test fails because returncode and regex specified         
    run: hostname                                                               
    status:                                                                     
      returncode: [0]                                                           
      regex:                                                                    
        stream: stdout                                                          
        exp: "^hello"                                                           
                                                                                
                                                                                
  status_runtime_min_max:                                                       
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if its within 1.0-
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: 1.0                                                                
        max: 4.0                                                                
                                                                                
  status_runtime_min:                                                           
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if exceeds mintime
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: 1.0                                                                
                                                                                
  status_runtime_max:                                                           
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if less than maxti
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        max: 4.0                                                                
                                                                                
  sbatch_example:                                                               
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using sbatch directives                
    sbatch:                                                                     
      - "-t 10:00:00"                                                           
      - "-p normal"                                                             
      - "-N 1"                                                                  
      - "-n 8"                                                                  
    run: hostname                                                               
                                                                                
  bsub_example:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using bsub directives                  
    bsub:                                                                       
      - "-W 00:30"                                                              
      - "-N 1"                                                                  
    run: hostname                                                               
                                                                                
  cobalt_example:                                                               
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using cobalt directives                
    cobalt:                                                                     
      - "-t 30"                                                                 
      - "-n 1"                                                                  
    run: hostname                                                               
                                                                                
  skip_example:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: this test is skip                                              
    skip: true                                                                  
    run: hostname                                                               
                                                                                
  summary_example:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: The summary field can be a multi-line string and exceed 80 char
    summary: |                                                                  
      This is a long description of test that                                   
      can exceed 80 characters and be multiline                                 
    run: hostname                                                               
                                                                                
  tag_str_example:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: tags can be defined as string                                  
    tags: network                                                               
    run: hostname                                                               
                                                                                
  tag_list_example:                                                             
    type: script                                                                
    executor: generic.local.bash                                                
    description: This is a tag example using list                               
    sbatch:                                                                     
      - "-t 10:00:00"                                                           
      - "-p normal"                                                             
      - "-N 1"                                                                  
      - "-n 8"                                                                  
    tags: ["slurm"]                                                             
    run: hostname                                                               
                                                                                
  metrics_regex_example:                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: metrics regular expression example                             
    run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"           
    metrics:                                                                    
      hpcg_rating:                                                              
        regex:                                                                  
          exp: 'rating of=(\d+\.\d+)$'                                          
          stream: stdout                                                        
                                                                                
  multi_executor_vars:                                                          
    type: script                                                                
    executor: 'generic.local.(sh|bash)'                                         
    description: single test multiple executor with variable declaration        
    run: |                                                                      
      echo $X                                                                   
      echo $Y                                                                   
    executors:                                                                  
      generic.local.sh:                                                         
        vars:                                                                   
          X: 1                                                                  
          Y: 2                                                                  
      generic.local.bash:                                                       
        vars:                                                                   
          X: 10                                                                 
          Y: 11                                                                 
                                                                                
  multi_executor_environment:                                                   
    type: script                                                                
    executor: 'generic.local.(sh|bash)'                                         
    description: single test multiple executor with environment declaration     
    run: echo $SHELL                                                            
    executors:                                                                  
      generic.local.sh:                                                         
        env:                                                                    
          SHELL: sh                                                             
      generic.local.bash:                                                       
        env:                                                                    
          SHELL: bash                                                           
                                                                                
  executors_sbatch_declaration:                                                 
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring sbatch by executors section                          
    tags: [tutorials]                                                           
    run: hostname                                                               
    sbatch: ["-N 4"]                                                            
    executors:                                                                  
      generic.local.bash:                                                       
        sbatch: ["-n 4", "-N 1", "-t 30"]                                       
      generic.local.sh:                                                         
        sbatch: ["-n 8", "-N 1", "-t 60"]                                       
                                                                                
  executors_bsub_declaration:                                                   
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring bsub  by executors section                           
    tags: [tutorials]                                                           
    run: hostname                                                               
    executors:                                                                  
      generic.local.bash:                                                       
        bsub: ["-n 4", "-W 30"]                                                 
      generic.local.sh:                                                         
        bsub: ["-n 8", "-W 60"]                                                 
                                                                                
  executors_pbs_declaration:                                                    
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring pbs by executors section                             
    tags: [tutorials]                                                           
    run: hostname                                                               
    executors:                                                                  
      generic.local.bash:                                                       
        pbs: ["-l ncpus=4", "-l walltime=30"]                                   
      generic.local.sh:                                                         
        pbs: ["-l ncpus=8", "-l walltime=60"]                                   
                                                                                
  executors_status_declaration:                                                 
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring status by executors section                          
    tags: [tutorials]                                                           
    run: exit 0                                                                 
    executors:                                                                  
      generic.local.bash:                                                       
        status:                                                                 
          returncode: 0                                                         
      generic.local.sh:                                                         
        status:                                                                 
          returncode: [1, 2]                                                    
                                                                                
  executors_metrics_declaration:                                                
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring metrics by executors section                         
    tags: [tutorials]                                                           
    run: echo "Hello World"                                                     
    executors:                                                                  
      generic.local.bash:                                                       
        metrics:                                                                
          hello:                                                                
            regex:                                                              
              stream: stdout                                                    
              exp: "(Hello)"                                                    
      generic.local.sh:                                                         
        metrics:                                                                
          world:                                                                
            regex:                                                              
              stream: stdout                                                    
              exp: "(World)"                                                    
                                                                                
  jobA:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: no job dependency                                              
    run: |                                                                      
      echo "This job has no dependency"                                         
      sleep 5                                                                   
                                                                                
  jobB:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: job dependency on A                                            
    needs: [jobA, jobB]                                                         
    run: |                                                                      
      echo "This job depends on jobA"                                           
      sleep 2

Compiler Schema Examples

buildtest schema -n compiler.schema.json --example
$ buildtest schema -n compiler.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  missing_type:                                                                 
    executor: local.bash                                                        
    description: "type key is missing, this is a required field"                
    source: "src/hello.c"                                                       
    compilers:                                                                  
      name: [intel]                                                             
                                                                                
  missing_required_compilers:                                                   
    executor: local.bash                                                        
    type: compiler                                                              
    description: "missing required field compilers "                            
    source: "src/hello.c"                                                       
                                                                                
  missing_required_source:                                                      
    executor: local.bash                                                        
    type: compiler                                                              
    description: "missing required field 'source' "                             
    compilers:                                                                  
      name: [gcc]                                                               
                                                                                
  invalid_type_value:                                                           
    executor: local.bash                                                        
    type: script                                                                
    description: "invalid value for type field must be 'compiler' "             
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
                                                                                
  invalid_description_value:                                                    
    executor: local.bash                                                        
    type: compiler                                                              
    description: 1                                                              
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
                                                                                
  invalid_type_module:                                                          
    executor: local.bash                                                        
    type: compiler                                                              
    description: "type for 'module' key, expecting a property but received 'stri
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module: "module load gcc/9.2.0"                                       
                                                                                
  module_purge_invalid_type:                                                    
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The purge property module is invalid. Expects bool got an int"
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            purge: 1                                                            
                                                                                
  module_swap_duplicate_check:                                                  
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The swap property expects two unique items"                   
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            swap: [gcc/8.0, gcc/8.0]                                            
                                                                                
  module_swap_min_items:                                                        
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The swap property expects a minimum of 2 items"               
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            swap: [gcc/8.0]                                                     
                                                                                
  module_swap_max_items:                                                        
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The swap property expects a maximum of 2 items"               
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            swap: [gcc/8.0, gcc/9.0, gcc/10.0]                                  
                                                                                
  module_load_duplicate_items:                                                  
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The load property expects unique items"                       
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            load: [gcc/9.2.0, gcc/9.2.0]                                        
                                                                                
  module_load_min_items:                                                        
    executor: local.bash                                                        
    type: compiler                                                              
    description: "The load property expects a minimum of 1 item"                
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc/9.2.0:                                                              
          module:                                                               
            load: []                                                            
                                                                                
  additionalProperties_main:                                                    
    executor: local.bash                                                        
    type: compiler                                                              
    description: "test additionalProperties in main schema"                     
    foo: bar                                                                    
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
                                                                                
  missing_required_compiler_name:                                               
    executor: local.bash                                                        
    type: compiler                                                              
    description: "'name' field in compilers section is required field"          
    source: src/hello.f90                                                       
    compilers:                                                                  
    default:                                                                    
      cray:                                                                     
        fflags: "-O1"                                                           
    config:                                                                     
      PrgEnv-cray@2.6.2:                                                        
        module:                                                                 
          swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                               
                                                                                
  uniqueItems_compiler_name:                                                    
    executor: local.bash                                                        
    type: compiler                                                              
    description: "Test unique items in 'name' field in compilers section"       
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["^(PrgEnv-cray)", "^(PrgEnv-cray)"]                                
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          fflags: "-O1"                                                         
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
                                                                                
  additionalProperties_compiler:                                                
    executor: local.bash                                                        
    type: compiler                                                              
    description: "Test additionalProperies in compiler section"                 
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["PrgEnv-cray"]                                                     
      FOO: BAR                                                                  
      default:                                                                  
        all:                                                                    
          env:                                                                  
            X: 1                                                                
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          fflags: "-O1"                                                         
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
                                                                                
  additionalProperties_compiler_default_all:                                    
    executor: local.bash                                                        
    type: compiler                                                              
    description: "Test additionalProperies in compiler default all section"     
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["PrgEnv-cray"]                                                     
      default:                                                                  
        all:                                                                    
          XYZ: 123                                                              
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          fflags: "-O1"                                                         
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
                                                                                
  additionalProperties_compiler_config:                                         
    executor: local.bash                                                        
    type: compiler                                                              
    description: "Test additionalProperies in compiler config section, FOO: BAR"
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["PrgEnv-cray"]                                                     
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          FOO: BAR                                                              
          fflags: "-O1"                                                         
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  gnu_example:                                                                  
    executor: local.bash                                                        
    type: compiler                                                              
    description: "gnu example with modules, and cflags example"                 
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [gcc]                                                               
      config:                                                                   
        gcc@8.4.0:                                                              
          cflags: "-O3"                                                         
                                                                                
  intel_example:                                                                
    executor: local.bash                                                        
    type: compiler                                                              
    description: "intel example using cflags"                                   
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [intel]                                                             
      config:                                                                   
        intel@2018:                                                             
          cflags: "-O1"                                                         
                                                                                
  clang_example:                                                                
    executor: local.bash                                                        
    type: compiler                                                              
    description: "clang example using cflags"                                   
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [clang]                                                             
      default:                                                                  
        clang:                                                                  
          cflags: "-O1"                                                         
      config:                                                                   
        clang@11:                                                               
          cflags: "-O2"                                                         
                                                                                
  upcxx_example:                                                                
    executor: local.bash                                                        
    type: compiler                                                              
    description: "upcxx compiler declaration in default and config section "    
    source: src/hello.c                                                         
    compilers:                                                                  
      name: [upcxx]                                                             
      default:                                                                  
        upcxx:                                                                  
          cflags: "-g aries"                                                    
      config:                                                                   
        upcxx@2020:                                                             
          cflags: "-O1 -g aries"                                                
                                                                                
  pgi_example:                                                                  
    executor: local.bash                                                        
    type: compiler                                                              
    description: "pgi example using cxxflags, ldflags in default and config sect
    source: src/hello.cpp                                                       
    compilers:                                                                  
      name: ["^(pgi|PrgEnv)"]                                                   
      default:                                                                  
        pgi:                                                                    
          cxxflags: "-O1"                                                       
          ldflags: "-lm"                                                        
      config:                                                                   
        pgi@18.1:                                                               
          module:                                                               
            swap: [PrgEnv-gnu, PrgEnv-pgi]                                      
            load: [pgi/18.1]                                                    
        pgi@18.2:                                                               
          module:                                                               
            swap: [PrgEnv-gnu, PrgEnv-pgi]                                      
            load: [pgi/18.2]                                                    
                                                                                
  cray_example:                                                                 
    executor: local.bash                                                        
    type: compiler                                                              
    description: "cray example using fflags and cppflags"                       
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["PrgEnv-cray"]                                                     
      default:                                                                  
        cray:                                                                   
          fflags: "-O1"                                                         
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
                                                                                
                                                                                
  sbatch_example_all_compiler_groups:                                           
    type: compiler                                                              
    description: sbatch example to for all compiler groups                      
    executor: local.bash                                                        
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["PrgEnv-cray"]                                                     
      default:                                                                  
        cray:                                                                   
          fflags: "-O1"                                                         
        all:                                                                    
          sbatch: ["-t 10", "-n 2", "-C haswell" ]                              
      config:                                                                   
        PrgEnv-cray@2.6.2:                                                      
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]                             
                                                                                
  bsub_all_compiler_groups:                                                     
    type: compiler                                                              
    description: bsub example for all compiler groups                           
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: [intel]                                                             
      default:                                                                  
        all:                                                                    
          bsub: ["-W 00:30", "-n 2"]                                            
      config:                                                                   
        intel@2019:                                                             
          cxxflags: "-O1"                                                       
                                                                                
  cobalt_all_compiler_groups:                                                   
    type: compiler                                                              
    description: cobalt example for all compiler groups                         
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: [intel]                                                             
      default:                                                                  
        all:                                                                    
          cobalt: ["-t 30", "-n 1"]                                             
      config:                                                                   
        intel@2019:                                                             
          cxxflags: "-O1"                                                       
                                                                                
  sbatch_compiler_group:                                                        
    type: compiler                                                              
    description: sbatch example in multiple compiler groups.                    
    executor: local.bash                                                        
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["^(gcc|intel)"]                                                    
      default:                                                                  
        gcc:                                                                    
          fflags: "-O1"                                                         
          sbatch: ["-t 10", "-n 2", "-C haswell" ]                              
        intel:                                                                  
          fflags: "-O2"                                                         
          sbatch: ["-t 10", "-n 2", "-C knl" ]                                  
      config:                                                                   
        gcc@8.1.0:                                                              
          sbatch: ["-t 60", "-n 2", "-C knl"]                                   
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]                              
                                                                                
                                                                                
  bsub_compiler_group:                                                          
    type: compiler                                                              
    description: bsub example in multiple compiler groups.                      
    executor: local.bash                                                        
    source: src/hello.f90                                                       
    compilers:                                                                  
      name: ["^(gcc|intel)"]                                                    
      default:                                                                  
        gcc:                                                                    
          fflags: "-O1"                                                         
          bsub: ["-W 00:30", "-n 2" ]                                           
        intel:                                                                  
          fflags: "-O2"                                                         
          bsub: ["-W 00:30", "-n 4" ]                                           
      config:                                                                   
        gcc@8.1.0:                                                              
           bsub: ["-W 00:30", "-n 6" ]                                          
           module:                                                              
             swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]                             
                                                                                
  env_example:                                                                  
    type: compiler                                                              
    description: Setting environment variables                                  
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: ["^(gcc)"]                                                          
      default:                                                                  
        all:                                                                    
          env:                                                                  
            OMP_NUM_THREADS: 2                                                  
          run: $_EXEC 1 2 4                                                     
      config:                                                                   
        gcc@10.2.0:                                                             
          cxxflags: "-fopenmp"                                                  
                                                                                
  custom_env_by_compiler_group:                                                 
    type: compiler                                                              
    description: Setting environment variables in compiler groups               
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: ["^(gcc|pgi)"]                                                      
      default:                                                                  
        all:                                                                    
          run: $_EXEC 1 2 4                                                     
        gcc:                                                                    
          cxxflags: "-fopenmp"                                                  
          env:                                                                  
            OMP_NUM_THREADS: 4                                                  
        pgi:                                                                    
          cxxflags: "-mp"                                                       
          env:                                                                  
            OMP_NUM_THREADS: 6                                                  
      config:                                                                   
        gcc@10.2.0:                                                             
          env:                                                                  
            OMP_NUM_THREADS: 6                                                  
                                                                                
        gcc@9.2.0:                                                              
          env:                                                                  
            OMP_NUM_THREADS: 8                                                  
                                                                                
        pgi@9.2.0:                                                              
          env:                                                                  
            OMP_NUM_THREADS: 10                                                 
                                                                                
  vars_example:                                                                 
    type: compiler                                                              
    description: Setting shell variables                                        
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: ["^(gcc)"]                                                          
      default:                                                                  
        all:                                                                    
          vars:                                                                 
            OUTFILE: /tmp/file1.txt                                             
          run: $_EXEC > $OUTFILE                                                
      config:                                                                   
        gcc@10.2.0:                                                             
          cxxflags: "-fopenmp"                                                  
                                                                                
  pass_args_run:                                                                
    type: compiler                                                              
    description: Passing arguments to executable in run section                 
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: [intel]                                                             
      default:                                                                  
        all:                                                                    
          run: $_EXEC 1 2 4                                                     
      config:                                                                   
        intel@2019:                                                             
          cxxflags: "-O1"                                                       
                                                                                
  mpi_launcher_example:                                                         
    type: compiler                                                              
    description: mpi launcher example                                           
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: [gcc]                                                               
      default:                                                                  
        all:                                                                    
          run: mpirun -np 2 $_EXEC                                              
      config:                                                                   
        gcc@7.3.0:                                                              
          cflags: "-O3"                                                         
          cxx: mpicxx                                                           
                                                                                
  status_returncode_example:                                                    
    type: compiler                                                              
    description: Status returncode match example                                
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: [gnu]                                                               
      default:                                                                  
        all:                                                                    
          vars:                                                                 
            OUTFILE: /tmp/file1.txt                                             
          run: $_EXEC > $OUTFILE                                                
          status:                                                               
            returncode: 1                                                       
      config:                                                                   
        gcc@10.2.0:                                                             
          cxxflags: "-fopenmp"                                                  
                                                                                
  pre_post_build_run_sections:                                                  
    type: compiler                                                              
    description: Run commands pre and post build section                        
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: ["^(gcc)"]                                                          
      default:                                                                  
        all:                                                                    
          pre_build: echo "pre-build section for ALL compilers"                 
          post_build: echo "post-build section for ALL Compilers"               
          pre_run: echo "pre-run section for ALL compilers"                     
          post_run: echo "post-run section for ALL Compilers"                   
        gcc:                                                                    
          pre_build: echo "pre-build section for GCC compilers"                 
          post_build: echo "post-build section for GCC compilers"               
          pre_run: echo "pre-run section for ALL compilers"                     
          post_run: echo "post-run section for ALL Compilers"                   
      config:                                                                   
        gcc@7.3.0:                                                              
          pre_build: echo "pre-build section for gcc@7.3.0"                     
          post_build: echo "post-build section for gcc@7.3.0"                   
          pre_run: echo "pre-run section for ALL compilers"                     
          post_run: echo "post-run section for ALL Compilers"                   
          cflags: "-O3"                                                         
        gcc@8.2.0:                                                              
          pre_build: echo "gcc --version"                                       
          cflags: "-O3"                                                         
                                                                                
                                                                                
  multi_compilers:                                                              
    type: compiler                                                              
    description: Select one or more compilers to run test                       
    executor: local.bash                                                        
    source: "src/hello.cpp"                                                     
    compilers:                                                                  
      name: ["^(gcc|intel|pgi|cray)"]                                           
      exclude: [intel@18]                                                       
      default:                                                                  
        gcc:                                                                    
          cflags: "-fopenmp"                                                    
        intel:                                                                  
          cflags: "-qopenmp"                                                    
        pgi:                                                                    
          cflags: "-fopenmp"                                                    
        cray:                                                                   
          cflags: "-h omp"                                                      
      config:                                                                   
        gcc@7.5.0:                                                              
          cflags: "-O3"                                                         
          module:                                                               
            load: [gcc/7.5.0]                                                   
        intel@17:                                                               
          module:                                                               
            load: [intel/2017]                                                  
        intel@18:                                                               
          module:                                                               
            load: [intel/2018]                                                  
        pgi/18.0:                                                               
          module:                                                               
            load: [pgi/18.0]                                                    
        craype/2.6.2:                                                           
          module:                                                               
            swap: [PrgEnv-intel, PrgEnv-cray]                                   
            load: [craype/2.6.2]

Spack Schema Examples

buildtest schema -n spack.schema.json --example
$ buildtest schema -n spack.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  specs_must_be_list_of_strings:                                                
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: 'specs must be a list of strings '                             
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - 1                                                                   
          - zlib                                                                
                                                                                
  additionalProperties_spack_field:                                             
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: additional Properties can't be specified in spack section      
    spack:                                                                      
      root: $HOME/spack                                                         
      FOO: BAR                                                                  
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - zlib                                                                
                                                                                
  invalid_type_mirror_field:                                                    
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: The mirror field must be a key value pair                      
    spack:                                                                      
      root: $HOME/spack                                                         
      mirror: https://caches.e4s.io                                             
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - zlib                                                                
                                                                                
  spack_test_additionalProperties:                                              
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "Check for additionalProperties in test section. FOO key is not
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib' ]                                                  
      test:                                                                     
        FOO: BAR                                                                
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 'zlib']                                                 
                                                                                
  spack_test_results_testing_required_oneOf:                                    
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "The results property expects one to specify specs, suite, or b
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib' ]                                                  
      test:                                                                     
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 'zlib']                                                 
        results:                                                                
          option: '-f'                                                          
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $SPACK_ROOT                                                        
                                                                                
  spack_test_run_invalid_spec:                                                  
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "specs property requires a list of strings. "                  
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib' ]                                                  
      test:                                                                     
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 1]                                                      
        results:                                                                
          option: '-f'                                                          
          suite: ['zlib']                                                       
          specs: ['m4']                                                         
                                                                                
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $SPACK_ROOT                                                        
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/devel/… ─
buildspecs:                                                                     
  env_create_name:                                                              
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack environment by name                               
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - m4                                                                  
          - zlib                                                                
      install:                                                                  
        options: ''                                                             
                                                                                
  env_activate:                                                                 
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Activate spack environment by name                             
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        activate:                                                               
          name: myproject                                                       
        specs:                                                                  
          - m4                                                                  
          - zlib                                                                
      install:                                                                  
        options: ''                                                             
                                                                                
  env_create_directory:                                                         
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack environment by directory                          
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          dir: $HOME/spack-env/myproject                                        
        specs:                                                                  
          - 'm4'                                                                
          - 'zlib@1.2.11'                                                       
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
  env_create_from_manifest:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack enviromment from manifest file                    
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
  env_concretized_install:                                                      
    type: spack                                                                 
    description: run 'spack concretize -f' in an environment and install specs  
    executor: generic.local.sh                                                  
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
        concretize: true                                                        
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
  env_mirror:                                                                   
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in e
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        mirror:                                                                 
          h5: /path/to/mirror                                                   
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
  install_without_env:                                                          
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: install specs without environment                              
    spack:                                                                      
      root: $HOME/spack/                                                        
      install:                                                                  
        options: '--cache-only'                                                 
        specs: ['m4', 'bzip2']                                                  
                                                                                
                                                                                
  pre_post_cmd_spack_install:                                                   
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: run commands before and after spack using pre_cmds and post_cmd
    pre_cmds: |                                                                 
      cd $HOME                                                                  
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: $HOME/spack/                                                        
      install:                                                                  
        options: '--cache-only'                                                 
        specs: ['m4', 'bzip2']                                                  
    post_cmds: |                                                                
      spack find                                                                
                                                                                
  remove_spack_environment:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: remove spack environment explicitly before creating environment
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        rm:                                                                     
          name: m4                                                              
        create:                                                                 
          name: m4                                                              
        activate:                                                               
          name: m4                                                              
        specs:                                                                  
          - 'm4'                                                                
                                                                                
  remove_spack_environment_automatically:                                       
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: remove spack environment automatically                         
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          remove_environment: true                                              
          name: m4                                                              
        activate:                                                               
          name: m4                                                              
        specs:                                                                  
          - 'm4'                                                                
                                                                                
                                                                                
  spack_test_results_specs_suites:                                              
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "Report results using suitename and spec format in spack test r
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib' ]                                                  
      test:                                                                     
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 'zlib']                                                 
        results:                                                                
          option: '-f'                                                          
          suite: ['zlib']                                                       
          specs: ['m4']                                                         
                                                                                
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $SPACK_ROOT                                                        
                                                                                
  sbatch_field_with_spack:                                                      
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Specify sbatch field with spack schema                         
    sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"]                             
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
  skip_test_in_spack:                                                           
    skip: True                                                                  
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: This test will be skipped                                      
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
  var_declaration_in_spack:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Define variables and environment variables in spack            
    vars:                                                                       
      FOO: "BAR"                                                                
    env:                                                                        
      SPACK_GNUPGHOME: "$HOME/.gnupg"                                           
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
  spack_sbatch_multi_executors:                                                 
    type: spack                                                                 
    executor: "generic.local.(sh|bash)"                                         
    description: "sbatch directives can be defined in spack schema"             
    tags: [spack]                                                               
    executors:                                                                  
      generic.local.sh:                                                         
        sbatch: ["-N 1", "-t 20"]                                               
      generic.local.bash:                                                       
        sbatch: ["-N 8", "-t 10"]                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        specs:                                                                  
          - 'm4'                                                                
        activate:                                                               
          name: m4                                                              
        concretize: true                                                        
                                                                                
  vars_multi_executors:                                                         
    type: spack                                                                 
    executor: "generic.local.(sh|bash)"                                         
    description: "variable declaration with multiple executors"                 
    tags: [spack]                                                               
    executors:                                                                  
      generic.local.sh:                                                         
        vars:                                                                   
          FOO: BAR                                                              
      generic.local.bash:                                                       
        vars:                                                                   
          HELLO: WORLD                                                          
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        specs:                                                                  
          - 'm4'                                                                
        activate:                                                               
          name: m4                                                              
        concretize: true