Buildtest Schemas¶
buildtest uses JSON Schema for validating buildspecs and buildtest configuration file. The json schemas are published at https://buildtesters.github.io/buildtest/ and we provide a command line interface to view schema files and examples.
Schema Naming Convention¶
All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.
CLI for buildtest schemas (buildtest schema)¶
You can use buildtest schema command to see the list of schemas
supported by buildtest. The schema files are denoted by .schema.json file extension.
buildtest schema
$ buildtest schema
global.schema.json
definitions.schema.json
settings.schema.json
compiler.schema.json
spack.schema.json
script.schema.json
Shown below is the command usage of buildtest schema
buildtest schema --help
$ buildtest schema --help
usage: buildtest [options] [COMMANDS] schema [-h] [-e] [-j] [-n Schema Name]
optional arguments:
-h, --help show this help message and exit
-e, --example Show schema examples
-j, --json Display json schema file
-n Schema Name, --name Schema Name
show schema by name (e.g., script)
You must use the --name option to select a schema, for instance if you want to view the JSON Schema for
script.schema.json you can run the following:
buildtest schema --name script.schema.json --json
Schema Files¶
Definition Schema¶
This schema is used for declaring definitions that need to be
reused in multiple schemas. We use $ref keyword to reference definitions from this file.
definitions.schema.json
{
"$id": "definitions.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "JSON Schema Definitions File. ",
"description": "This file is used for declaring definitions that are referenced from other schemas",
"definitions": {
"list_of_strings": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"string_or_list": {
"oneOf": [
{
"type": "string"
},
{
"$ref": "#/definitions/list_of_strings"
}
]
},
"list_of_ints": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "integer"
}
},
"int_or_list": {
"oneOf": [
{
"type": "integer"
},
{
"$ref": "#/definitions/list_of_ints"
}
]
},
"file_regex": {
"type": "array",
"description": "Specify a list of regular expressions to match files in the current working directory. The regular expression is matched using ``re.search`` python module. ",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"file",
"exp"
],
"properties": {
"file": {
"type": "string",
"description": "Specify a file name to match with regular expression"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run on the selected file name"
}
}
}
},
"file_regex_in_metrics": {
"type": "object",
"additionalProperties": false,
"description": "Specify a regular expressions on a filepath used for assigning value to metrics. The regular expression is matched using ``re.search`` python module. ",
"required": [
"file",
"exp"
],
"properties": {
"additionalProperties": false,
"file": {
"type": "string",
"description": "Specify a file name to match with regular expression"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run on the selected file name"
},
"item": {
"type": "integer",
"minimum": 0,
"description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
}
}
},
"regex": {
"type": "object",
"additionalProperties": false,
"description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
"required": [
"exp",
"stream"
],
"properties": {
"stream": {
"type": "string",
"enum": [
"stdout",
"stderr"
],
"description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
},
"item": {
"type": "integer",
"minimum": 0,
"description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
}
}
},
"env": {
"type": "object",
"description": "One or more key value pairs for an environment (key=value)",
"minItems": 1,
"items": {
"type": "object",
"minItems": 1,
"propertyNames": {
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
}
}
},
"description": {
"type": "string",
"description": "The ``description`` field is used to document what the test is doing",
"maxLength": 80
},
"summary": {
"type": "string",
"description": "The ``summary`` field is used to document what the test is doing and can be a multi-line string"
},
"tags": {
"description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
"$ref": "#/definitions/string_or_list"
},
"skip": {
"type": "boolean",
"description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
},
"executor": {
"type": "string",
"description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
},
"needs": {
"description": "A list of test names that are dependency before runnning job",
"type": "array",
"items": [
{
"oneOf": [
{ "type": "string" },
{
"type": "object",
"patternProperties": {
"^.*$": {
"additionalProperties": false,
"properties": {
"state": {"$ref": "#/definitions/state"},
"returncode": {"$ref": "#/definitions/returncode"}
}
}
}
}
]
}
]
},
"metrics_field": {
"type": "object",
"additionalProperties": false,
"required": [
"type"
],
"properties": {
"type": {
"type": "string",
"description": "Specify python data-type (str, int, float) to convert metric. ",
"enum": [
"str",
"int",
"float"
]
},
"regex": {
"$ref": "#/definitions/regex"
},
"file_regex": {
"$ref": "#/definitions/file_regex_in_metrics"
}
},
"oneOf": [
{ "required": ["regex" ] },
{ "required": [ "file_regex"] }
]
},
"metrics": {
"type": "object",
"description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/metrics_field"
}
},
"state": {
"type": "string",
"description": "explicitly mark state of test regardless of status calculation",
"enum": [
"PASS",
"FAIL"
]
},
"returncode": {
"description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
"$ref": "#/definitions/int_or_list"
},
"status": {
"type": "object",
"description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
"additionalProperties": false,
"properties": {
"slurm_job_state": {
"type": "string",
"enum": [
"COMPLETED",
"FAILED",
"OUT_OF_MEMORY",
"TIMEOUT"
],
"description": "This field can be used to pass test based on Slurm Job State, if there is a match buildtest will report as ``PASS`` "
},
"pbs_job_state":
{
"type": "string",
"description": "This field can be used to pass test based on PBS Job State, if there is a match buildtest will report as ``PASS`` ",
"enum": [
"H",
"S",
"F"
]
},
"lsf_job_state": {
"type": "string",
"description": "This field can be used to pass test based on LSF Job State, if there is a match buildtest will report as ``PASS`` ",
"enum": [
"DONE",
"EXIT"
]
},
"returncode": { "$ref": "#/definitions/returncode" },
"regex": {
"$ref": "#/definitions/regex",
"description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
},
"file_regex": {
"$ref": "#/definitions/file_regex",
"description": "Perform a regular expression search on a list of arbitrary files, if expression matches for any file buildtest will report as ``PASS``. This field is useful for checking output of a test that is not captured in stdout or stderr. "
},
"runtime": {
"type": "object",
"description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
"properties": {
"min": {
"type": "number",
"minimum": 0,
"description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
},
"max": {
"type": "number",
"minimum": 0,
"description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
}
}
},
"assert_ge": {
"type": "array",
"description": "Perform assertion of greater and equal (>=) with reference value",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "number",
"description": "Specify reference value (int,float) for comparison"
}
}
}
},
"assert_le": {
"type": "array",
"description": "Perform assertion of less than and equal (<=) with reference value",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "number",
"description": "Specify reference value (int,float) for comparison"
}
}
}
},
"assert_gt": {
"type": "array",
"description": "Perform assertion of greater than (>) with reference value",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "number",
"description": "Specify reference value (int,float) for comparison"
}
}
}
},
"assert_lt": {
"type": "array",
"description": "Perform assertion of less than (<) with reference value",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "number",
"description": "Specify reference value (int,float) for comparison"
}
}
}
},
"assert_eq":
{
"description": "Perform assertion of equality (=) with reference value",
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"description": "Specify reference value (str, int,float) for comparison",
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
}
}
}
},
"assert_ne":
{
"description": "Perform assertion of not equal with reference value",
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"description": "Specify reference value (str, int, float) for comparison",
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
}
}
}
},
"assert_range":
{
"description": "Perform assertion based on lower and upper bound",
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"lower",
"upper"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"lower": {
"description": "Specify reference value for lower bound",
"type": "number"
},
"upper": {
"description": "Specify reference value for upper bound",
"type": "number"
}
}
}
},
"contains": {
"description": "Check if metric value is in a list of reference values ",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "array",
"minItems": 1,
"description": "Specify a list of reference value"
}
}
}
},
"not_contains": {
"description": "Check if metric value not in a list of reference values ",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "array",
"minItems": 1,
"description": "Specify a list of reference value"
}
}
}
},
"is_symlink": {
"description": "Check for list of files or directory paths that are symbolic links",
"$ref": "#/definitions/list_of_strings"
},
"exists": {
"description": "Check for list of file or directory path for existences using os.path.exists",
"$ref": "#/definitions/list_of_strings"
},
"is_dir": {
"description": "Check for list of filepaths are directories",
"$ref": "#/definitions/list_of_strings"
},
"is_file": {
"description": "Check for list of filepaths are files",
"$ref": "#/definitions/list_of_strings"
},
"file_count": {
"description": "Perform assertion check by comparing file count in a directory",
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"dir",
"count"
],
"properties": {
"dir": {
"type": "string",
"description": "Directory path to check for file count"
},
"count": {
"type": "integer",
"description": "Number of files expected in directory",
"minimum": 0
},
"depth": {
"type": "integer",
"description": "Depth of directory to search for files",
"minimum": 0
},
"ext": {
"$ref": "#/definitions/string_or_list",
"description": "File extension to filter files in directory"
},
"filepattern": {
"type": "string",
"description": "Specify a regular expression when searching for files in directory"
},
"filetype": {
"type": "string",
"description": "Specify file type when searching for files in directory. It can be 'file', 'dir' or 'symlink'",
"enum": [
"file",
"dir",
"symlink"
]
},
"file_traverse_limit": {
"type": "integer",
"description": "Limit the number of files to traverse when searching for files in directory",
"minimum": 1,
"default": 10000,
"maximum": 999999
}
}
}
},
"state": {
"$ref": "#/definitions/state",
"description": "explicitly mark state of test regardless of status calculation"
},
"mode": {
"type": "string",
"description": "Determine how the status check is resolved, for instance it can be logical AND or OR",
"enum": ["any", "all"]
}
}
},
"BB": {
"$ref": "#/definitions/list_of_strings",
"description": "Create burst buffer space, this specifies #BB options in your test."
},
"DW": {
"$ref": "#/definitions/list_of_strings",
"description": "Specify Data Warp option (#DW) when using burst buffer."
},
"sbatch": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #SBATCH options in test script."
},
"bsub": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #BSUB options in test script."
},
"cobalt": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #COBALT options in test script."
},
"pbs": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #PBS directives in test script."
},
"executors": {
"type": "object",
"description": "Define executor specific configuration",
"patternProperties": {
"description": "Name of executor to override configuration",
"^.*$": {
"additionalProperties": false,
"properties": {
"env": {
"$ref": "#/definitions/env"
},
"vars": {
"$ref": "#/definitions/env"
},
"sbatch": {
"$ref": "#/definitions/list_of_strings"
},
"bsub": {
"$ref": "#/definitions/list_of_strings"
},
"pbs": {
"$ref": "#/definitions/list_of_strings"
},
"cobalt": {
"$ref": "#/definitions/list_of_strings"
},
"BB": {
"$ref": "#/definitions/BB"
},
"DW": {
"$ref": "#/definitions/DW"
},
"status": {
"$ref": "#/definitions/status"
},
"metrics": {
"$ref": "#/definitions/metrics"
}
}
}
}
},
"cc": {
"type": "string",
"description": "Set C compiler wrapper"
},
"fc": {
"type": "string",
"description": "Set Fortran compiler wrapper"
},
"cxx": {
"type": "string",
"description": "Set C++ compiler wrapper"
},
"cflags": {
"type": "string",
"description": "Set C compiler flags."
},
"fflags": {
"type": "string",
"description": "Set Fortran compiler flags."
},
"cxxflags": {
"type": "string",
"description": "Set C++ compiler flags."
},
"ldflags": {
"type": "string",
"description": "Set linker flags"
},
"cppflags": {
"type": "string",
"description": "Set C or C++ preprocessor flags"
},
"run": {
"type": "string",
"description": "Specify a series of commands to run."
},
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"restore": {
"description": "Load a collection name via ``module restore``",
"type": "string"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": {
"type": "string"
}
}
}
}
}
}
Settings Schema¶
This schema defines how buildtest configuration file is validated.
settings.schema.json
{
"$id": "settings.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "buildtest configuration schema",
"type": "object",
"required": [
"system"
],
"additionalProperties": false,
"properties": {
"system": {
"type": "object",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/system"
}
}
}
},
"definitions": {
"terse": {
"type": "boolean",
"description": "A boolean to determine whether to enable terse mode"
},
"format": {
"type": "string",
"description": "Determine the format fields to display when viewing table results"
},
"count": {
"type": "integer",
"minimum": 1,
"description": "Determine number of records to display in a table"
},
"system": {
"required": [
"executors",
"moduletool",
"hostnames",
"compilers",
"buildspecs",
"report"
],
"additionalProperties": false,
"type": "object",
"properties": {
"hostnames": {
"type": "array",
"description": "Specify a list of hostnames to check where buildtest can run for the given system record",
"items": {
"type": "string"
}
},
"description": {
"type": "string",
"description": "system description field"
},
"poolsize":
{
"type": "integer",
"minimum": 1,
"description": "Specify size of Process Pool for parallel processing using ``multiprocessing.Pool``"
},
"testdir": {
"type": "string",
"description": "Specify full path to test directory where buildtest will write tests."
},
"logdir": {
"type": "string",
"description": "Specify location where buildtest will write log files"
},
"moduletool": {
"type": "string",
"description": "Specify modules tool used for interacting with ``module`` command. ",
"enum": [
"environment-modules",
"lmod",
"N/A"
]
},
"timeout": {
"type": "integer",
"minimum": 1,
"description": "Specify timeout duration in number of seconds"
},
"pager":
{
"type": "boolean",
"description": "A boolean to determine whether to enable paging when viewing buildspec cache"
},
"buildspecs":
{
"type": "object",
"description": "Specify configuration for ``buildtest buildspec`` command",
"additionalProperties": false,
"properties": {
"rebuild": {
"type": "boolean",
"description": "A boolean to determine whether to rebuild buildspec cache"
},
"count": { "$ref": "#/definitions/count" } ,
"format": {
"$ref": "#/definitions/format"
},
"terse": { "$ref": "#/definitions/terse" },
"root": {
"type": "array",
"items": {
"type": "string"
},
"description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
}
}
},
"report":
{
"type": "object",
"properties": {
"terse": { "$ref": "#/definitions/terse" },
"format": { "$ref": "#/definitions/format" },
"count": { "$ref": "#/definitions/count" },
"latest": {"type": "boolean", "description": "A boolean to determine whether to show latest test run"},
"oldest": {"type": "boolean", "description": "A boolean to determine whether to show oldest test run"}
}
},
"processor": {
"type": "object",
"description": "Specify processor information",
"additionalProperties": false,
"properties": {
"numcpus": {"type": "integer", "minimum": 1, "description": "Specify Total Number of CPUs"},
"sockets": {"type": "integer", "minimum": 1, "description": "Specify Number of CPU Sockets"},
"cores": {"type": "integer", "minimum": 1, "description": "Specify Number of Physical Cores"},
"threads_per_core": {"type": "integer", "minimum": 1, "description": "Specify Threads per Core" },
"core_per_socket": {"type": "integer", "minimum": 1, "description": "Specify Cores per Socket"},
"model": {"type": "string", "description": "Specify Processor Model" },
"arch": {"type": "string", "description": "Specify processor architecture"},
"vendor": {"type": "string", "description": "Vendor Name"}
}
},
"compilers": {
"type": "object",
"description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
"additionalProperties": false,
"properties": {
"enable_prgenv": {
"type": "boolean",
"description": "Enable support for Programming Environment"
},
"modulepath": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"purge": {"type": "boolean", "description": "A boolean to determine whether to purge modules via ``module purge`` when generating compiler declaration"},
"prgenv_modules": {
"type": "object",
"additionalProperties": false,
"properties": {
"gcc": {
"type": "string",
"description": "Specify name of Programming Environment module for gcc",
"enum": [
"PrgEnv-gnu"
]
},
"intel": {
"type": "string",
"description": "Specify name of Programming Environment module for intel",
"enum": [
"PrgEnv-intel"
]
},
"cray": {
"type": "string",
"description": "Specify name of Programming Environment module for cray",
"enum": [
"PrgEnv-cray"
]
},
"nvhpc": {
"type": "string",
"description": "Specify name of Programming Environment module for nvhpc",
"enum": [
"PrgEnv-nvhpc",
"PrgEnv-nvidia"
]
}
}
},
"find": {
"type": "object",
"additionalProperties": false,
"description": "Find compilers by specifying regular expression that is applied to modulefile names",
"properties": {
"gcc": {
"type": "string",
"description": "Specify a regular expression to search for gcc compilers from your module stack"
},
"intel": {
"type": "string",
"description": "Specify a regular expression to search for intel compilers from your module stack"
},
"cray": {
"type": "string",
"description": "Specify a regular expression to search for cray compilers from your module stack"
},
"clang": {
"type": "string",
"description": "Specify a regular expression to search for clang compilers from your module stack"
},
"cuda": {
"type": "string",
"description": "Specify a regular expression to search for cuda compilers from your module stack"
},
"pgi": {
"type": "string",
"description": "Specify a regular expression to search for pgi compilers from your module stack"
},
"upcxx": {
"type": "string",
"description": "Specify a regular expression to search for upcxx compilers from your module stack"
},
"nvhpc": {
"type": "string",
"description": "Specify a regular expression to search for nvhpc compilers from your module stack"
}
}
},
"compiler": {
"type": "object",
"additionalProperties": false,
"description": "Start of compiler declaration",
"properties": {
"gcc": {
"description": "Declaration of one or more GNU compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"intel": {
"description": "Declaration of one or more Intel compilers. ",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cray": {
"description": "Declaration of one or more Cray compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"pgi": {
"description": "Declaration of one or more PGI compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"clang": {
"description": "Declaration of one or more Clang compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cuda": {
"description": "Declaration of one or more CUDA compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"upcxx": {
"description": "Declaration of one or more UPCXX compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"nvhpc": {
"description": "Declaration of one or more NVHPC compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
}
}
}
}
},
"executors": {
"type": "object",
"additionalProperties": false,
"description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``, ``cobalt``. The executors are referenced in buildspec using ``executor`` field.",
"required": ["local"],
"properties": {
"defaults": {
"type": "object",
"description": "Specify default executor settings for all executors",
"additionalProperties": false,
"properties": {
"pollinterval": {
"type": "integer",
"description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
"minimum": 1,
"default": 30
},
"maxpendtime": {
"$ref": "#/definitions/maxpendtime"
},
"account": {
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"}
}
},
"local": {
"type": "object",
"description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/local"
}
},
"lsf": {
"type": "object",
"description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/lsf"
}
},
"slurm": {
"type": "object",
"description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/slurm"
}
},
"cobalt": {
"type": "object",
"description": "The ``cobalt`` section is used for declaring Cobalt executors for running jobs using Cobalt scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/cobalt"
}
},
"pbs": {
"type": "object",
"description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/pbs"
}
}
}
},
"cdash": {
"type": "object",
"description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
"required": ["url", "project", "site"],
"properties": {
"url": {
"type": "string",
"description": "Url to CDASH server"
},
"project": {
"type": "string",
"description": "Name of CDASH project"
},
"site": {
"type": "string",
"description": "Site Name reported in CDASH"
}
}
}
}
},
"cc": {
"description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
"type": "string"
},
"cxx": {
"type": "string",
"description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"fc": {
"type": "string",
"description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"max_jobs": {
"description": "Maximum number of jobs that can be run at a given time for a particular executor",
"type": "integer",
"minimum": 1
},
"compiler_section": {
"description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
"type": "object",
"additionalProperties": false,
"required": [ "cc", "cxx", "fc" ],
"properties": {
"cc": { "$ref": "#/definitions/cc" },
"cxx": { "$ref": "#/definitions/cxx" },
"fc": { "$ref": "#/definitions/fc" },
"module": { "$ref": "#/definitions/module" }
}
},
"unique_string_array": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"disable": {"type": "boolean", "description": "Disable executor"},
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": { "type": "string" }
},
"restore": {
"description": "Load a collection name via ``module restore``",
"type": "string"
}
}
},
"script": {
"type": "array",
"additionalProperties": false,
"items": { "type": "string" }
},
"maxpendtime": {
"type": "integer",
"description": "Cancel job if it is still pending in queue beyond maxpendtime",
"minimum": 1,
"default": 86400
},
"account": {
"type": "string",
"description": "Specify Job Account for charging resources"
},
"local": {
"type": "object",
"description": "An instance object of local executor",
"additionalProperties": false,
"required": [ "shell" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"shell": {
"type": "string",
"description": "Specify the shell launcher you want to use when running tests locally"
},
"before_script": { "#ref": "#/definitions/script" },
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
},
"slurm": {
"type": "object",
"additionalProperties": false,
"description": "An instance object of slurm executor",
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "sbatch" ],
"description": "Specify the slurm batch scheduler to use. This overrides the default ``launcher`` field. This must be ``sbatch``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
},
"cluster": {
"type": "string",
"description": "Specify the slurm cluster you want to use ``-M <cluster>``"
},
"partition": {
"type": "string",
"description": "Specify the slurm partition you want to use ``-p <partition>``"
},
"qos": {
"type": "string",
"description": "Specify the slurm qos you want to use ``-q <qos>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
},
"lsf": {
"type": "object",
"description": "An instance object of lsf executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "bsub" ],
"description": "Specify the lsf batch scheduler to use. This overrides the default ``launcher`` field. It must be ``bsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
},
"cobalt": {
"type": "object",
"description": "An instance object of cobalt executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "qsub" ],
"description": "Specify the cobalt batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
},
"pbs": {
"type": "object",
"description": "An instance object of cobalt executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "qsub" ],
"description": "Specify the pbs batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
}
}
}
Global Schema¶
This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema
global.schema.json
{
"$id": "global.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "global schema",
"description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
"type": "object",
"required": ["buildspecs"],
"additionalProperties": false,
"properties": {
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"maintainers": {
"type": "array",
"description": "One or more maintainers or aliases",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"buildspecs": {
"type": "object",
"description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
"propertyNames": {
"pattern": "^[A-Za-z_.-][A-Za-z0-9_.-]*$",
"maxLength": 48
}
}
}
}
Script Schema¶
This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when
type: script is specified. For more details on script schema see Script Schema.
script.schema.json
{
"$id": "script.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "script schema version",
"description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
"type": "object",
"required": [
"type",
"run",
"executor"
],
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^script$",
"description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
},
"description": {
"$ref": "definitions.schema.json#/definitions/description"
},
"summary": {
"$ref": "definitions.schema.json#/definitions/summary"
},
"sbatch": {
"$ref": "definitions.schema.json#/definitions/sbatch"
},
"bsub": {
"$ref": "definitions.schema.json#/definitions/bsub"
},
"cobalt": {
"$ref": "definitions.schema.json#/definitions/cobalt"
},
"pbs": {
"$ref": "definitions.schema.json#/definitions/pbs"
},
"BB": {
"$ref": "definitions.schema.json#/definitions/BB"
},
"DW": {
"$ref": "definitions.schema.json#/definitions/DW"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"executor": {
"$ref": "definitions.schema.json#/definitions/executor"
},
"needs": {
"$ref": "definitions.schema.json#/definitions/needs"
},
"shell": {
"type": "string",
"description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed"
},
"shebang": {
"type": "string",
"description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"tags": {
"$ref": "definitions.schema.json#/definitions/tags"
},
"metrics": {
"$ref": "definitions.schema.json#/definitions/metrics"
},
"executors": {
"$ref": "definitions.schema.json#/definitions/executors"
},
"compilers": {
"type": "object",
"required": [
"name"
],
"additionalProperties": false,
"properties": {
"name": {
"description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"exclude": {
"description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"default": {
"type": "object",
"additionalProperties": false,
"properties": {
"gcc": {
"$ref": "#definitions/default_compiler_config"
},
"intel": {
"$ref": "#definitions/default_compiler_config"
},
"pgi": {
"$ref": "#definitions/default_compiler_config"
},
"cray": {
"$ref": "#definitions/default_compiler_config"
},
"clang": {
"$ref": "#definitions/default_compiler_config"
},
"cuda": {
"$ref": "#definitions/default_compiler_config"
},
"upcxx": {
"$ref": "#definitions/default_compiler_config"
},
"nvhpc": {
"$ref": "#definitions/default_compiler_config"
}
}
},
"config": {
"type": "object",
"description": "Specify compiler configuration based on named compilers.",
"patternProperties": {
"^.*$": {
"$ref": "#definitions/compiler_declaration"
}
}
}
}
}
},
"definitions": {
"compiler_declaration": {
"type": "object",
"description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
"additionalProperties": false,
"properties": {
"cc": {
"$ref": "definitions.schema.json#/definitions/cc"
},
"fc": {
"$ref": "definitions.schema.json#/definitions/fc"
},
"cxx": {
"$ref": "definitions.schema.json#/definitions/cxx"
},
"cflags": {
"$ref": "definitions.schema.json#/definitions/cflags"
},
"fflags": {
"$ref": "definitions.schema.json#/definitions/fflags"
},
"cxxflags": {
"$ref": "definitions.schema.json#/definitions/cxxflags"
},
"ldflags": {
"$ref": "definitions.schema.json#/definitions/ldflags"
},
"cppflags": {
"$ref": "definitions.schema.json#/definitions/cppflags"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
},
"module": {
"$ref": "definitions.schema.json#/definitions/module"
}
},
"default_compiler_config": {
"type": "object",
"description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
"properties": {
"cc": {
"$ref": "definitions.schema.json#/definitions/cc"
},
"fc": {
"$ref": "definitions.schema.json#/definitions/fc"
},
"cxx": {
"$ref": "definitions.schema.json#/definitions/cxx"
},
"cflags": {
"$ref": "definitions.schema.json#/definitions/cflags"
},
"fflags": {
"$ref": "definitions.schema.json#/definitions/fflags"
},
"cxxflags": {
"$ref": "definitions.schema.json#/definitions/cxxflags"
},
"ldflags": {
"$ref": "definitions.schema.json#/definitions/ldflags"
},
"cppflags": {
"$ref": "definitions.schema.json#/definitions/cppflags"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
}
}
}
},
"default_compiler_config": {
"type": "object",
"description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
"properties": {
"cc": {
"$ref": "definitions.schema.json#/definitions/cc"
},
"fc": {
"$ref": "definitions.schema.json#/definitions/fc"
},
"cxx": {
"$ref": "definitions.schema.json#/definitions/cxx"
},
"cflags": {
"$ref": "definitions.schema.json#/definitions/cflags"
},
"fflags": {
"$ref": "definitions.schema.json#/definitions/fflags"
},
"cxxflags": {
"$ref": "definitions.schema.json#/definitions/cxxflags"
},
"ldflags": {
"$ref": "definitions.schema.json#/definitions/ldflags"
},
"cppflags": {
"$ref": "definitions.schema.json#/definitions/cppflags"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
}
}
}
}
}
Compiler Schema¶
This is the compiler schema used for validating buildspecs that define test using type: compiler.
This schema is used for compiling a single source code. For more details see Using Compiler Schema
compiler.schema.json
{
"$id": "compiler.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "compiler schema",
"description": "The compiler schema is of ``type: compiler`` in sub-schema which is used for compiling and running programs",
"type": "object",
"required": [
"type",
"source",
"compilers",
"executor"
],
"definitions": {
"cc": {
"type": "string",
"description": "Set C compiler wrapper"
},
"fc": {
"type": "string",
"description": "Set Fortran compiler wrapper"
},
"cxx": {
"type": "string",
"description": "Set C++ compiler wrapper"
},
"cflags": {
"type": "string",
"description": "Set C compiler flags."
},
"fflags": {
"type": "string",
"description": "Set Fortran compiler flags."
},
"cxxflags": {
"type": "string",
"description": "Set C++ compiler flags."
},
"ldflags": {
"type": "string",
"description": "Set linker flags"
},
"cppflags": {
"type": "string",
"description": "Set C or C++ preprocessor flags"
},
"pre_build": {
"type": "string",
"description": "Run commands before building program"
},
"post_build": {
"type": "string",
"description": "Run commands after building program"
},
"pre_run": {
"type": "string",
"description": "Run commands before running program"
},
"post_run": {
"type": "string",
"description": "Run commands after running program"
},
"run": {
"type": "string",
"description": "Run command for launching compiled binary"
},
"default_compiler_all": {
"type": "object",
"description": "Specify compiler configuration for all compiler groups. Use the ``all`` property if configuration is shared across compiler groups. This property can be overridden in compiler group or named compiler in ``config`` section.",
"additionalProperties": false,
"properties": {
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"BB": { "$ref": "definitions.schema.json#/definitions/BB" },
"DW": { "$ref": "definitions.schema.json#/definitions/DW" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" }
}
},
"default_compiler_config": {
"type": "object",
"description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
"properties": {
"cc": { "$ref": "#definitions/cc" },
"fc": { "$ref": "#definitions/fc" },
"cxx": { "$ref": "#definitions/cxx" },
"cflags": { "$ref": "#definitions/cflags" },
"fflags": { "$ref": "#definitions/fflags" },
"cxxflags": { "$ref": "#definitions/cxxflags" },
"ldflags": { "$ref": "#definitions/ldflags" },
"cppflags": { "$ref": "#definitions/cppflags" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs"},
"BB": { "$ref": "definitions.schema.json#/definitions/BB"},
"DW": { "$ref": "definitions.schema.json#/definitions/DW"},
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" }
}
},
"compiler_declaration": {
"type": "object",
"description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
"additionalProperties": false,
"properties": {
"cc": { "$ref": "#definitions/cc" },
"fc": { "$ref": "#definitions/fc" },
"cxx": { "$ref": "#definitions/cxx" },
"cflags": { "$ref": "#definitions/cflags" },
"fflags": { "$ref": "#definitions/fflags" },
"cxxflags": { "$ref": "#definitions/cxxflags" },
"ldflags": { "$ref": "#definitions/ldflags" },
"cppflags": { "$ref": "#definitions/cppflags" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs"},
"BB": { "$ref": "definitions.schema.json#/definitions/BB"},
"DW": { "$ref": "definitions.schema.json#/definitions/DW"},
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" },
"module": { "$ref": "definitions.schema.json#/definitions/module" }
}
}
},
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^compiler$",
"description": "Select schema type to use when validating buildspec. This must be of set to ``compiler``."
},
"description": {
"$ref": "definitions.schema.json#/definitions/description"
},
"summary": {
"$ref": "definitions.schema.json#/definitions/summary"
},
"needs": {
"$ref": "definitions.schema.json#/definitions/needs"
},
"compilers": {
"type": "object",
"required": [
"name"
],
"additionalProperties": false,
"properties": {
"name": {
"description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"exclude": {
"description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"default": {
"type": "object",
"additionalProperties": false,
"properties": {
"all": {
"$ref": "#definitions/default_compiler_all"
},
"gcc": {
"$ref": "#definitions/default_compiler_config"
},
"intel": {
"$ref": "#definitions/default_compiler_config"
},
"pgi": {
"$ref": "#definitions/default_compiler_config"
},
"cray": {
"$ref": "#definitions/default_compiler_config"
},
"clang": {
"$ref": "#definitions/default_compiler_config"
},
"cuda": {
"$ref": "#definitions/default_compiler_config"
},
"upcxx": {
"$ref": "#definitions/default_compiler_config"
},
"nvhpc": {
"$ref": "#definitions/default_compiler_config"
}
}
},
"config": {
"type": "object",
"description": "Specify compiler configuration based on named compilers.",
"patternProperties": {
"^.*$": {
"$ref": "#definitions/compiler_declaration"
}
}
}
}
},
"source": {
"type": "string",
"description": "Specify a source file for compilation, the file can be relative path to buildspec or an absolute path"
},
"executor": {
"$ref": "definitions.schema.json#/definitions/executor"
},
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"tags": {
"$ref": "definitions.schema.json#/definitions/tags"
},
"metrics": {
"$ref": "definitions.schema.json#/definitions/metrics"
}
}
}
Spack Schema¶
This schema is used for writing tests with spack package manager using type: spack field. For more details
see Buildtest Spack Integration.
spack.schema.json
{
"$id": "spack.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "spack schema version",
"description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
"type": "object",
"required": [
"type",
"executor",
"spack"
],
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^spack$",
"description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
},
"description": { "$ref": "definitions.schema.json#/definitions/description" },
"summary": { "$ref": "definitions.schema.json#/definitions/summary" },
"executor": { "$ref": "definitions.schema.json#/definitions/executor" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"BB": { "$ref": "definitions.schema.json#/definitions/BB" },
"DW": { "$ref": "definitions.schema.json#/definitions/DW" },
"skip": { "$ref": "definitions.schema.json#/definitions/skip" },
"tags": { "$ref": "definitions.schema.json#/definitions/tags" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
"executors": { "$ref": "definitions.schema.json#/definitions/executors" },
"pre_cmds": {
"type": "string",
"description": "Shell commands run before spack"
},
"post_cmds": {
"type": "string",
"description": "Shell commands run after spack"
},
"needs": {
"$ref": "definitions.schema.json#/definitions/needs"
},
"spack": {
"type": "object",
"description": "Entry point to spack configuration",
"additionalProperties": false,
"properties": {
"root": {
"type": "string",
"description": "Specify location for root of spack directory"
},
"compiler_find": {
"type": "boolean",
"description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror by running ``spack mirror add``"
},
"env": {
"$ref": "#definitions/env",
"description": "Manage spack environments via ``spack env`` command"
},
"install": {
"$ref": "#definitions/install",
"description": "Install spack packages by running ``spack install``. "
},
"verify_spack": {
"type": "boolean",
"description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
"default": true
},
"test": {
"$ref": "#definitions/test",
"description": "Entry point to ``spack test``"
}
}
}
},
"definitions": {
"env": {
"additionalProperties": false,
"type": "object",
"description": "Used for managing spack environment using ``spack env`` command. ",
"properties": {
"create": {
"additionalProperties": false,
"description": "Create a spack environment via ``spack env create``",
"type": "object",
"properties": {
"remove_environment": {
"type": "boolean",
"description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
"default": false
},
"name": {
"type": "string",
"description": "Name of spack environment to create"
},
"manifest": {
"type": "string",
"description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
},
"options": {
"type": "string",
"description": "Pass options to ``spack env create`` command"
},
"dir": {
"type": "string",
"description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
}
}
},
"activate": {
"additionalProperties": false,
"type": "object",
"description": "Activate a spack environment via ``spack env activate``",
"properties": {
"name": {
"type": "string",
"description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
},
"options": {
"type": "string",
"description": "Pass options to ``spack env activate`` command"
},
"dir": {
"type": "string",
"description": "Activate spack environment from directory."
}
}
},
"rm": {
"additionalProperties": false,
"description": "Remove an existing spack environment via ``spack env rm``.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
}
}
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror in spack environment by running ``spack mirror add``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
},
"concretize": {
"type": "boolean",
"description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
}
}
},
"install": {
"description": "Install spack packages using ``spack install`` command",
"type": "object",
"properties": {
"options": {
"type": "string",
"description": "Pass options to ``spack install`` command"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to install using ``spack install`` command"
}
}
},
"test": {
"type": "object",
"additionalProperties": false,
"required": [
"run",
"results"
],
"properties": {
"remove_tests": {
"type": "boolean",
"description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
},
"run": {
"description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
"type": "object",
"required": [
"specs"
],
"additionalProperties": false,
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test run``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to run tests by running ``spack test run <specs>``."
}
}
},
"results": {
"type": "object",
"description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
"additionalProperties": false,
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test results``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Report result by spec name by running ``spack test run -- <specs>``."
}
}
}
}
}
}
}
Schema Examples¶
The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.
In order to view an example you can run:
buildtest schema -n <schema> --example
Settings Schema Examples¶
buildtest schema -n settings.schema.json --example
$ buildtest schema -n settings.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: N/A
executors:
local:
# the line below is invalid due to pattern property
==bash:
description: submit jobs on local machine
shell: bash -v
slurm:
# the line below is invalid due to pattern property
$haswell:
launcher: sbatch
options: ["-p haswell", "-t 00:10"]
lsf:
<>batch:
launcher: bsub
queue: batch
options: ["-q batch", "-t 00:10"]
cobalt:
<normal>:
launcher: qsub
queue: normal
options: ["-n 1", "-t 10"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: lmod
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
executors:
defaults:
pollinterval: 10
maxpendtime: 45
local:
bash:
description: submit jobs via bash shell
shell: bash
lsf:
batch:
description: "LSF Executor name 'batch' that submits jobs to 'batch' q
queue: batch
account: developer
options: ["-W 20"]
before_script: |
time
echo "commands run before job"
test:
description: "LSF Executor name 'test' that submits jobs to 'test' que
launcher: bsub
queue: test
account: qa
options: ["-W 20"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: lmod
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
testdir: /tmp/buildtest
executors:
defaults:
pollinterval: 20
maxpendtime: 30
account: admin
local:
bash:
description: submit jobs via bash shell
shell: bash
slurm:
normal:
options: ["-C haswell"]
qos: normal
before_script: |
time
echo "commands run before job"
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: N/A
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
executors:
defaults:
pollinterval: 10
maxpendtime: 30
local:
bash:
description: submit jobs via bash shell
shell: bash
pbs:
workq:
queue: workq
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: N/A
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
executors:
local:
bash:
description: submit jobs on local machine
shell: bash -v
slurm:
haswell:
launcher: sbatch
options: ["-p haswell", "-t 00:10"]
lsf:
batch:
launcher: bsub
queue: batch
options: ["-q batch", "-t 00:10"]
cobalt:
normal:
launcher: qsub
queue: normal
options: ["-n 1", "-t 10"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
#logdir: $BUILDTEST_ROOT/logs
#testdir: $BUILDTEST_ROOT/tests
moduletool: N/A
poolsize: 1
cdash:
url: https://my.cdash.org
project: buildtest
site: laptop
processor:
numcpus: 8
cores: 4
threads_per_core: 2
sockets: 1
model: "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz"
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
executors:
local:
bash:
description: submit jobs on local machine using bash shell
shell: bash
module:
purge: True
restore: "Default"
load: ["gcc"]
sh:
description: submit jobs on local machine using sh shell
shell: sh
before_script: |
date
echo "these commands will be run"
csh:
description: submit jobs on local machine using csh shell
shell: csh -x
tcsh:
description: submit jobs on local machine using tcsh shell
shell: /bin/tcsh
zsh:
description: submit jobs on local machine using zsh shell
shell: /bin/zsh
compilers:
find:
gcc: "^(gcc|GCC|PrgEnv-gnu)"
intel: "^(intel|Intel|PrgEnv-intel)"
cray: "^(cray|PrgEnv-cray)"
clang: "^(clang|Clang)"
cuda: "^(cuda|CUDA)"
pgi: "^(pgi|PGI|PrgEnv-pgi)"
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
gcc@7.2.0:
cc: 'cc'
cxx: 'cxx'
fc: 'fc'
module:
load:
- gcc/7.2.0
intel:
intel@2019:
cc: 'icc'
cxx: 'icpc'
fc: 'ifort'
module:
purge: True
load:
- gcc/7.2.0
- intel/2019
cray:
craype@2.6.2:
cc: 'cc'
cxx: 'CC'
fc: 'fc'
module:
load: [craype/2.6.2]
swap: [PrgEnv-gnu, PrgEnv-cray]
clang:
clang@12.0.0:
cc: 'clang'
cxx: 'clang++'
fc: 'None'
module:
load: [clang/12.0]
cuda:
cuda@11.0:
cc: 'nvcc'
cxx: 'nvcc'
fc: 'None'
module:
load: [cuda/11.0]
pgi:
pgi@18.0:
cc: 'pgcc'
cxx: 'pgc++'
fc: 'pgfortran'
module:
load: [pgi/18.0]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: lmod
poolsize: 1
executors:
defaults:
maxpendtime: 30
local:
bash:
description: submit jobs via bash shell
shell: bash
cobalt:
knl:
queue: knl
haswell:
queue: haswell
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
# show the latest for every test
latest: True
# show the oldest for every test
oldest: False
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
Global Schema Examples¶
buildtest schema -n global.schema.json --example
$ buildtest schema -n global.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
hostname:
type: script
run: "hostname"
maintainers: [shahzebsiddiqui, shahzebsiddiqui]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
# wrong type for maintainers key, expects a string
maintainers: 1
buildspecs:
hostname:
type: script
run: "hostname"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
# this test fails because it exceeds 32 character length for test name
buildspecs:
_________this_test_exceeds_character_length______________:
type: script
run: hostname
executor: generic.local.bash
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
# invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$"
(badname:
type: script
run: "ping login 1"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
# testing all caps
ABCDEFGHIJKLMNOPQRSTUVWXYZ:
type: script
run: "hostname"
# testing all lowercase letters
abcdefghijklmnopqrstuvwxyz:
type: script
run: "hostname"
# testing '_' in beginning followed by all numbers
_0123456789:
type: script
run: "hostname"
# testing '_' in combination with caps, lowercase and numbers
_ABCDEFabcdef0123456789:
type: script
run: "hostname"
# testing '_' at end of word
abcdefghijklmnopqrstuvwxyz_:
type: script
run: "hostname"
# testing '.' in beginning of word
.helloworld:
type: script
run: hostname
# testing '.' in middle of word
hello.world:
type: script
run: hostname
# testing '.' at end of word
helloworld.:
type: script
run: hostname
# testing '-' in middle of word
hello-world:
type: script
run: hostname
# testing '-' at end of word
helloworld-:
type: script
run: hostname
Script Schema Examples¶
buildtest schema -n script.schema.json --example
$ buildtest schema -n script.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_test_name_&!@#$%:
type: script
executor: generic.local.bash
description: "invalid test name"
invalid_bash:
type: script
executor: generic.local.bash
shell: "bash-missing-run"
missing_run_key:
type: script
executor: generic.local.bash
description: invalid key name roon, missing run key
roon: |
systemctl is-active slurmd
systemctl is-enabled slurmd | grep enabled
invalid_env_type:
type: script
executor: generic.local.bash
description: env key should be a dictionary
env:
- FOO=BAR
run: echo $FOO
invalid_vars_type:
type: script
executor: generic.local.bash
description: var key should be a dictionary
vars:
- FOO=BAR
run: echo $FOO
invalid_description:
type: script
executor: generic.local.bash
description:
- "Multi Line description"
- "is not accepted"
invalid_summary:
type: script
executor: generic.local.bash
description: summary field must be a string
summary: 1
invalid_regex_stream:
type: script
executor: generic.local.bash
description: This test fails because of invalid regex stream
run: hostname
status:
regex:
stream: file
exp: "world$"
regex_additionalProperties_test:
type: script
executor: generic.local.bash
description: Testing for additional properties in regex field
run: hostname
status:
regex:
stream: stdout
exp: "world$"
X: 1
missing_regex_exp:
type: script
executor: generic.local.bash
description: This test fails because of missing key 'exp' in regex
run: hostname
status:
regex:
stream: stdout
invalid_returncode_type:
type: script
executor: generic.local.bash
description: This test fails because of invalid return code type
run: hostname
status:
returncode: ["1"]
empty_returncode_list:
type: script
executor: generic.local.bash
description: An empty returncode list will cause an error
run: hostname
status:
returncode: []
non_int_returncodes:
type: script
executor: generic.local.bash
description: The returncode must be an int and not a number
run: exit 1
status:
returncode: 1.01
non_int_returncodes_list:
type: script
executor: generic.local.bash
description: The returncode must be a list of integers and no numbers
run: exit 1
status:
returncode: [1, 2.230]
invalid_shell_type:
type: script
executor: generic.local.bash
description: invalid shell type must be a string
shell: ["/bin/bash"]
run: hostname
invalid_type_shell_shebang:
type: script
executor: generic.local.bash
description: invalid type for shell shebang, must be a string
shebang: ["#!/bin/bash"]
run: hostname
invalid_skip_value:
type: script
executor: generic.local.bash
description: invalid value for skip, must be boolean
skip: 1
run: hostname
empty_tags:
type: script
executor: generic.local.bash
description: tag list can't be empty, requires one item.
tags: []
run: hostname
non_unique_tags:
type: script
executor: local.bash
description: tag names must be unique
tags: ["network", "network"]
run: hostname
invalid_tags_value:
type: script
executor: generic.local.bash
description: invalid tag value must be all string items
tags: ["network", 400]
run: hostname
additionalProperties_test:
type: script
executor: generic.local.bash
description: additional properties are not allowed so any invalid key/value
FOO: BAR
run: hostname
additionalProperties_status:
type: script
executor: generic.local.bash
description: test additional properties in status object. This is not allowe
sbatch: ["-n 2", "-q normal", "-t 10"]
run: hostname
status:
slurm_job_state: "COMPLETED"
FOO: BAR
invalid_runtime_min:
type: script
executor: generic.local.sh
description: "Invalid type for min property in runtime. Must be an integer o
run: sleep 2
status:
runtime:
min: "1"
runtime_min_must_exceed_0:
type: script
executor: generic.local.sh
description: "The runtime must exceed 0"
run: sleep 2
status:
runtime:
min: -1
invalid_slurm_job_state:
type: script
executor: generic.local.sh
description: invalid value for slurm_job_state, should raise error with enum
sbatch:
- "-n 2"
- "-q normal"
- "-t 10"
run: hostname
status:
slurm_job_state: "FINISH"
invalid_metrics_additional_property:
type: script
executor: generic.local.bash
description: Test for additional property for metrics property
vars:
FOO: BAR
run: echo $FOO
metrics:
foo:
variable: FOO
invalid_metrics_name:
type: script
executor: generic.local.bash
description: "Metrics name does not follow pattern"
run: echo "hello" > file.txt
metrics:
(foo-bar:
type: str
regex:
stream: stdout
exp: "BAR"
missing_regex_in_metrics:
type: script
executor: generic.local.bash
description: Test for missing regex in metrics property, which should result
run: echo "hello"
metrics:
foo:
type: str
test_regex_and_file_regex_in_metrics:
type: script
executor: generic.local.bash
description: Test for regex and file_regex in metrics. Only one is allowed a
run: echo "hello" > file.txt
metrics:
foo:
type: str
file_regex:
file: file.txt
exp: "hello"
regex:
stream: stdout
exp: "hello"
invalid_metrics_type:
type: script
executor: generic.local.bash
description: metrics property is an object, testing for type
vars:
FOO: BAR
run: echo $FOO
metrics: FOO
metric_type_property_test:
type: script
executor: generic.local.bash
description: metrics 'type' must be a 'str', 'float', 'int'
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: tuple
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
executors_invalid_var_type:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Invalid type field for 'vars'
tags: [tutorials]
run: echo $FOO
executors:
generic.local.bash:
vars: ["FOO=BAR"]
executors_additionalProperties:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Testing for additional properties in 'executors'
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
FOO: BAR
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
generic.local.zsh:
sbatch: ["-n 16", "-N 2", "-t 120"]
file_exists_failure:
type: script
executor: generic.local.bash
description: this test will fail validation, because item must be a string
run: mkdir -p 1
status:
exists: [1]
invalid_property_filename:
type: script
executor: generic.local.bash
description: Invalid property name filename under file_regex
run: |
echo "Hello" > hello.txt
status:
file_regex:
- filename: hello.txt
exp: '^(Hello)$'
testing_additionalProperties_file_regex:
type: script
executor: generic.local.bash
description: Testing for additional properties in file_regex
run: |
echo "Hello" > hello.txt
status:
file_regex:
- file: hello.txt
exp: '^(Hello)$'
FOO: BAR
file_count_property_count_failure:
type: script
executor: generic.local.bash
description: The count property in file_count must be greater than 0
run: hostname
status:
file_count:
- dir: /tmp
count: -1
file_count_property_depth_failure:
type: script
executor: generic.local.bash
description: The depth property in file_count must be greater than 0
run: hostname
status:
file_count:
- dir: /tmp
count: 0
depth: -1
file_count_required_property_check:
type: script
executor: generic.local.bash
description: The dir property in file_count is required
run: hostname
status:
file_count:
- count: 0
file_count_additionalProperties_check:
type: script
executor: generic.local.bash
description: Testing for additional properties in file_count
run: hostname
status:
file_count:
- dir: /tmp
count: 0
FOO: BAR
file_count_minimum_check_on_count:
type: script
executor: generic.local.bash
description: The count property must be 0 or greater
run: mkdir foo
status:
file_count:
- dir: foo
count: -1
file_count_filetype_invalid_value:
type: script
executor: generic.local.bash
description: Invalid value for filetype, it must be 'file', 'dir' or 'symlin
run: mkdir foo
status:
file_count:
- dir: foo
count: -1
filetype: 'filename'
minimum_check_file_traverse_limit:
type: script
executor: generic.local.bash
description: The minimum value for file_traverse_limit is 1
run: mkdir foo
status:
file_count:
- dir: foo
count: 1
file_traverse_limit: 0
maximum_check_file_traverse_limit:
type: script
executor: generic.local.bash
description: The minimum value for file_traverse_limit is 999999
run: mkdir foo
status:
file_count:
- dir: foo
count: 1
file_traverse_limit: 1000000
invalid_value_for_mode:
type: script
executor: generic.local.bash
description: The status mode must be 'any' or 'all'
run: exit 0
status:
returncode: 0
mode: '1'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
multiline_run:
executor: generic.local.bash
type: script
description: multiline run command
run: |
echo "1"
echo "2"
single_command_run:
executor: generic.local.bash
type: script
description: single command as a string for run command
run: "hostname"
declare_env:
executor: generic.local.bash
type: script
description: declaring environment variables
env:
FOO: BAR
X: 1
run: |
echo $FOO
echo $X
declare_vars:
executor: generic.local.bash
type: script
description: declaring variables
vars:
First: Bob
Last: Bill
run: |
echo "First:" $First
echo "Last:" $Last
declare_shell_sh:
executor: generic.local.sh
type: script
description: declare shell name to sh
shell: sh
run: hostname
declare_shell_bash:
executor: generic.local.bash
type: script
description: declare shell name to bash
shell: bash
run: hostname
declare_shell_python:
executor: generic.local.bash
type: script
description: declare shell name to python
shell: python
run: |
print("Hello World")
declare_shell_bin_bash:
executor: generic.local.bash
type: script
description: declare shell name to /bin/bash
shell: "/bin/bash -e"
run: hostname
declare_shell_name_bin_sh:
executor: generic.local.sh
type: script
description: declare shell name to /bin/sh
shell: "/bin/sh -e"
run: hostname
declare_shell_opts:
executor: generic.local.sh
type: script
description: declare shell name to sh
shell: "sh -e"
run: hostname
declare_shell_bin_zsh:
executor: generic.local.zsh
type: script
description: declare shell zsh
shell: "zsh"
run: hostname
declare_shell_zsh:
executor: generic.local.zsh
type: script
description: declare shell /bin/zsh
shell: "zsh"
run: hostname
declare_shell_bin_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/csh
shell: "/bin/csh"
run: hostname
declare_shell_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "csh"
run: hostname
declare_shell_bin_tcsh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "/bin/tcsh"
run: hostname
declare_shell_tcsh:
executor: generic.local.csh
type: script
description: declare shell tcsh
shell: "tcsh"
run: hostname
declare_shebang:
executor: generic.local.bash
type: script
description: declare shell name to sh
shebang: "#!/usr/bin/env bash"
run: hostname
status_returncode_list:
executor: generic.local.bash
type: script
description: The returncode can be a list of integers
run: exit 0
status:
returncode: [0]
status_returncode_int:
executor: generic.local.bash
type: script
description: The returncode can be an integer to match with single returncod
run: exit 0
status:
returncode: 0
status_regex:
executor: generic.local.bash
type: script
description: This test pass with a regular expression status check
run: hostname
status:
regex:
stream: stdout
exp: "^$"
status_regex_returncode:
executor: generic.local.bash
type: script
description: This test fails because returncode and regex specified
run: hostname
status:
returncode: [0]
regex:
stream: stdout
exp: "^hello"
status_runtime_min_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if its within 1.0-
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
max: 4.0
status_runtime_min:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if exceeds mintime
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
status_runtime_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if less than maxti
tags: ["tutorials"]
run: sleep 2
status:
runtime:
max: 4.0
sbatch_example:
type: script
executor: generic.local.bash
description: This test runs hostname using sbatch directives
sbatch:
- "-t 10:00:00"
- "-p normal"
- "-N 1"
- "-n 8"
run: hostname
bsub_example:
type: script
executor: generic.local.bash
description: This test runs hostname using bsub directives
bsub:
- "-W 00:30"
- "-N 1"
run: hostname
cobalt_example:
type: script
executor: generic.local.bash
description: This test runs hostname using cobalt directives
cobalt:
- "-t 30"
- "-n 1"
run: hostname
skip_example:
type: script
executor: generic.local.bash
description: this test is skip
skip: true
run: hostname
summary_example:
type: script
executor: generic.local.bash
description: The summary field can be a multi-line string and exceed 80 char
summary: |
This is a long description of test that
can exceed 80 characters and be multiline
run: hostname
tag_str_example:
type: script
executor: generic.local.bash
description: tags can be defined as string
tags: network
run: hostname
tag_list_example:
type: script
executor: generic.local.bash
description: This is a tag example using list
sbatch:
- "-t 10:00:00"
- "-p normal"
- "-N 1"
- "-n 8"
tags: ["slurm"]
run: hostname
metrics_regex_example:
type: script
executor: generic.local.bash
description: metrics regular expression example
run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
metrics:
hpcg_rating:
type: float
regex:
exp: 'rating of=(\d+\.\d+)$'
stream: stdout
metrics_regex_with_file:
type: script
executor: generic.local.bash
description: metrics regular expression example
run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" > hpcg.txt
metrics:
hpcg_rating:
type: float
file_regex:
exp: 'rating of=(\d+\.\d+)$'
file: hpcg.txt
multi_executor_vars:
type: script
executor: 'generic.local.(sh|bash)'
description: single test multiple executor with variable declaration
run: |
echo $X
echo $Y
executors:
generic.local.sh:
vars:
X: 1
Y: 2
generic.local.bash:
vars:
X: 10
Y: 11
multi_executor_environment:
type: script
executor: 'generic.local.(sh|bash)'
description: single test multiple executor with environment declaration
run: echo $SHELL
executors:
generic.local.sh:
env:
SHELL: sh
generic.local.bash:
env:
SHELL: bash
executors_sbatch_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring sbatch by executors section
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
executors_bsub_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring bsub by executors section
tags: [tutorials]
run: hostname
executors:
generic.local.bash:
bsub: ["-n 4", "-W 30"]
generic.local.sh:
bsub: ["-n 8", "-W 60"]
executors_pbs_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring pbs by executors section
tags: [tutorials]
run: hostname
executors:
generic.local.bash:
pbs: ["-l ncpus=4", "-l walltime=30"]
generic.local.sh:
pbs: ["-l ncpus=8", "-l walltime=60"]
executors_status_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring status by executors section
tags: [tutorials]
run: exit 0
executors:
generic.local.bash:
status:
returncode: 0
generic.local.sh:
status:
returncode: [1, 2]
executors_metrics_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring metrics by executors section
tags: [tutorials]
run: echo "Hello World"
executors:
generic.local.bash:
metrics:
hello:
type: str
regex:
stream: stdout
exp: "(Hello)"
generic.local.sh:
metrics:
world:
type: str
regex:
stream: stdout
exp: "(World)"
jobA:
type: script
executor: generic.local.bash
description: no job dependency
run: |
echo "This job has no dependency"
sleep 5
jobB:
type: script
executor: generic.local.bash
description: job dependency on A
needs: [jobA, jobB]
run: |
echo "This job depends on jobA"
sleep 2
assert_range_ex:
type: script
executor: generic.local.bash
description: Example on assert_range
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_range:
- name: copy
lower: 5000
upper: 20000
- name: scale
lower: 4500
upper: 20000
assert_eq_example:
type: script
executor: generic.local.bash
description: Test for assert equality
vars:
X: 1
Y: 1.5
first: John
last: Smith
run: |
echo "X: $X"
echo "Y: $Y"
echo "Name: $first $last"
metrics:
x:
type: int
regex:
stream: stdout
exp: 'X:\s+(\S+)\s+.*'
item: 1
y:
type: float
regex:
stream: stdout
exp: 'Y:\s+(\S+)\s+.*'
item: 1
first:
type: str
regex:
stream: stdout
exp: 'Name:\s+(\S+)\s+.*'
item: 1
last:
type: str
regex:
stream: stdout
exp: '(Smith)$'
item: 1
status:
assert_eq:
- name: x
ref: 1
- name: y
ref: 1.5
- name: first
ref: John
- name: last
ref: Smith
stream_test_assert_ge:
type: script
executor: generic.local.bash
description: Run stream test with metrics example using assert greater equal
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_ge:
- name: copy
ref: 5000
- name: scale
ref: 5500
- name: add
ref: 6000
- name: triad
ref: 6500
stream_test_assert_ge_exception:
type: script
executor: generic.local.bash
description: Run stream test with testing exception for assert_ge
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: str
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_ge:
- name: copy
ref: 5000
file_exists_pass:
type: script
executor: generic.local.bash
description: this test will pass
run: mkdir -p 1
status:
exists: ['1']
combined_file_and_dir_checks:
type: script
executor: generic.local.bash
description: status check for files and directories
run: hostname
status:
is_dir:
- $HOME
- /tmp
is_file:
- $HOME/.bashrc
is_symlink_example:
type: script
executor: generic.local.bash
description: status check based on symbolic link
run: |
ln -s /tmp scratch
ln -s $HOME/.bash_profile $HOME/bash_profile
status:
is_symlink:
- scratch
- $HOME/bash_profile
- ~/bash_profile
regex_on_multiple_files:
type: script
executor: generic.local.bash
description: Test regex on multiple files
run: |
echo "Hello" > hello.txt
buildtest --help > buildtest_help.txt
status:
file_regex:
- file: hello.txt
exp: '^(Hello)$'
- file: buildtest_help.txt
exp: '^(usage: buildtest)'
regex_on_directory_not_supported:
type: script
executor: generic.local.bash
description: Test regex on directory is not supported
run: |
mkdir -p hello
echo "Hello" > hello/hello.txt
status:
file_regex:
- file: hello
exp: '^(Hello)$'
file_expansion_supported:
type: script
executor: generic.local.bash
description: Test regex with variable and shell expansion
run: |
echo "Hello" > $BUILDTEST_ROOT/hello.txt
echo "Hello" > $HOME/hello.txt
status:
file_regex:
- file: $BUILDTEST_ROOT/hello.txt
exp: '^(Hello)$'
- file: ~/hello.txt
exp: '^(Hello)$'
file_count_on_directory:
type: script
executor: generic.local.bash
description: file count check in directory
run: |
mkdir -p foo
touch foo/{1..5}
status:
file_count:
- dir: foo
count: 5
file_count_by_extension:
type: script
executor: generic.local.bash
description: file count by extension
run: |
mkdir -p foo/bar
touch foo/{1..5}.sh
touch foo/bar/{1..3}.py foo/bar/{1..3}.txt
status:
file_count:
- dir: foo
ext: '.sh'
depth: 1
count: 5
- dir: foo/bar
ext: ['.py', '.txt']
count: 6
file_extension_and_filepattern:
type: script
executor: generic.local.bash
description: file count by file extension and file pattern
run: |
touch foo{1..5}.txt
touch {conf,main}.py
ls -l
status:
file_count:
- dir: .
ext: '.txt'
filepattern: '(conf|main).py$'
count: 7
file_count_by_filetype:
type: script
executor: generic.local.bash
description: Count the number of directories and symbolic links
run: |
mkdir -p foo/{bar,baz}
find foo -type dir
ln -s foo/bar foo/bar.link
ln -s foo/baz foo/baz.link
status:
file_count:
- dir: foo
count: 3
filetype: 'dir'
- dir: foo
count: 2
filetype: 'symlink'
file_traverse_limit:
type: script
executor: generic.local.bash
description: Use of file_traverse_limit to limit number of files searched in
run: |
mkdir foo
touch foo/{1..99}.txt
status:
file_count:
- dir: foo
count: 50
file_traverse_limit: 50
- dir: foo
count: 10
file_traverse_limit: 20
status_logical_and:
type: script
executor: 'generic.local.bash'
description: 'Using logical AND to check status'
run: |
echo "This is a test"
exit 1
status:
mode: all
returncode: 1
regex:
stream: stdout
exp: 'This is a test'
status_logical_or:
type: script
executor: 'generic.local.bash'
description: 'Using logical OR to check status'
run: |
echo "This is a test"
exit 1
status:
mode: any
returncode: 0
regex:
stream: stdout
exp: 'This is a test'
Compiler Schema Examples¶
buildtest schema -n compiler.schema.json --example
$ buildtest schema -n compiler.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
missing_type:
executor: local.bash
description: "type key is missing, this is a required field"
source: "src/hello.c"
compilers:
name: [intel]
missing_required_compilers:
executor: local.bash
type: compiler
description: "missing required field compilers "
source: "src/hello.c"
missing_required_source:
executor: local.bash
type: compiler
description: "missing required field 'source' "
compilers:
name: [gcc]
invalid_type_value:
executor: local.bash
type: script
description: "invalid value for type field must be 'compiler' "
source: src/hello.c
compilers:
name: [gcc]
invalid_description_value:
executor: local.bash
type: compiler
description: 1
source: src/hello.c
compilers:
name: [gcc]
invalid_type_module:
executor: local.bash
type: compiler
description: "type for 'module' key, expecting a property but received 'stri
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module: "module load gcc/9.2.0"
module_purge_invalid_type:
executor: local.bash
type: compiler
description: "The purge property module is invalid. Expects bool got an int"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
purge: 1
module_swap_duplicate_check:
executor: local.bash
type: compiler
description: "The swap property expects two unique items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0, gcc/8.0]
module_swap_min_items:
executor: local.bash
type: compiler
description: "The swap property expects a minimum of 2 items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0]
module_swap_max_items:
executor: local.bash
type: compiler
description: "The swap property expects a maximum of 2 items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0, gcc/9.0, gcc/10.0]
module_load_duplicate_items:
executor: local.bash
type: compiler
description: "The load property expects unique items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
load: [gcc/9.2.0, gcc/9.2.0]
module_load_min_items:
executor: local.bash
type: compiler
description: "The load property expects a minimum of 1 item"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
load: []
additionalProperties_main:
executor: local.bash
type: compiler
description: "test additionalProperties in main schema"
foo: bar
source: src/hello.c
compilers:
name: [gcc]
missing_required_compiler_name:
executor: local.bash
type: compiler
description: "'name' field in compilers section is required field"
source: src/hello.f90
compilers:
default:
cray:
fflags: "-O1"
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
uniqueItems_compiler_name:
executor: local.bash
type: compiler
description: "Test unique items in 'name' field in compilers section"
source: src/hello.f90
compilers:
name: ["^(PrgEnv-cray)", "^(PrgEnv-cray)"]
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler section"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
FOO: BAR
default:
all:
env:
X: 1
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler_default_all:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler default all section"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
all:
XYZ: 123
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler_config:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler config section, FOO: BAR"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
config:
PrgEnv-cray@2.6.2:
FOO: BAR
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
gnu_example:
executor: local.bash
type: compiler
description: "gnu example with modules, and cflags example"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc@8.4.0:
cflags: "-O3"
intel_example:
executor: local.bash
type: compiler
description: "intel example using cflags"
source: src/hello.c
compilers:
name: [intel]
config:
intel@2018:
cflags: "-O1"
clang_example:
executor: local.bash
type: compiler
description: "clang example using cflags"
source: src/hello.c
compilers:
name: [clang]
default:
clang:
cflags: "-O1"
config:
clang@11:
cflags: "-O2"
upcxx_example:
executor: local.bash
type: compiler
description: "upcxx compiler declaration in default and config section "
source: src/hello.c
compilers:
name: [upcxx]
default:
upcxx:
cflags: "-g aries"
config:
upcxx@2020:
cflags: "-O1 -g aries"
pgi_example:
executor: local.bash
type: compiler
description: "pgi example using cxxflags, ldflags in default and config sect
source: src/hello.cpp
compilers:
name: ["^(pgi|PrgEnv)"]
default:
pgi:
cxxflags: "-O1"
ldflags: "-lm"
config:
pgi@18.1:
module:
swap: [PrgEnv-gnu, PrgEnv-pgi]
load: [pgi/18.1]
pgi@18.2:
module:
swap: [PrgEnv-gnu, PrgEnv-pgi]
load: [pgi/18.2]
cray_example:
executor: local.bash
type: compiler
description: "cray example using fflags and cppflags"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
cray:
fflags: "-O1"
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
sbatch_example_all_compiler_groups:
type: compiler
description: sbatch example to for all compiler groups
executor: local.bash
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
cray:
fflags: "-O1"
all:
sbatch: ["-t 10", "-n 2", "-C haswell"]
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
bsub_all_compiler_groups:
type: compiler
description: bsub example for all compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
bsub: ["-W 00:30", "-n 2"]
config:
intel@2019:
cxxflags: "-O1"
cobalt_all_compiler_groups:
type: compiler
description: cobalt example for all compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
cobalt: ["-t 30", "-n 1"]
config:
intel@2019:
cxxflags: "-O1"
sbatch_compiler_group:
type: compiler
description: sbatch example in multiple compiler groups.
executor: local.bash
source: src/hello.f90
compilers:
name: ["^(gcc|intel)"]
default:
gcc:
fflags: "-O1"
sbatch: ["-t 10", "-n 2", "-C haswell"]
intel:
fflags: "-O2"
sbatch: ["-t 10", "-n 2", "-C knl"]
config:
gcc@8.1.0:
sbatch: ["-t 60", "-n 2", "-C knl"]
module:
swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]
bsub_compiler_group:
type: compiler
description: bsub example in multiple compiler groups.
executor: local.bash
source: src/hello.f90
compilers:
name: ["^(gcc|intel)"]
default:
gcc:
fflags: "-O1"
bsub: ["-W 00:30", "-n 2"]
intel:
fflags: "-O2"
bsub: ["-W 00:30", "-n 4"]
config:
gcc@8.1.0:
bsub: ["-W 00:30", "-n 6"]
module:
swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]
env_example:
type: compiler
description: Setting environment variables
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
env:
OMP_NUM_THREADS: 2
run: $_EXEC 1 2 4
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
custom_env_by_compiler_group:
type: compiler
description: Setting environment variables in compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc|pgi)"]
default:
all:
run: $_EXEC 1 2 4
gcc:
cxxflags: "-fopenmp"
env:
OMP_NUM_THREADS: 4
pgi:
cxxflags: "-mp"
env:
OMP_NUM_THREADS: 6
config:
gcc@10.2.0:
env:
OMP_NUM_THREADS: 6
gcc@9.2.0:
env:
OMP_NUM_THREADS: 8
pgi@9.2.0:
env:
OMP_NUM_THREADS: 10
vars_example:
type: compiler
description: Setting shell variables
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
vars:
OUTFILE: /tmp/file1.txt
run: $_EXEC > $OUTFILE
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
pass_args_run:
type: compiler
description: Passing arguments to executable in run section
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
run: $_EXEC 1 2 4
config:
intel@2019:
cxxflags: "-O1"
mpi_launcher_example:
type: compiler
description: mpi launcher example
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [gcc]
default:
all:
run: mpirun -np 2 $_EXEC
config:
gcc@7.3.0:
cflags: "-O3"
cxx: mpicxx
status_returncode_example:
type: compiler
description: Status returncode match example
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [gnu]
default:
all:
vars:
OUTFILE: /tmp/file1.txt
run: $_EXEC > $OUTFILE
status:
returncode: 1
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
pre_post_build_run_sections:
type: compiler
description: Run commands pre and post build section
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
pre_build: echo "pre-build section for ALL compilers"
post_build: echo "post-build section for ALL Compilers"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
gcc:
pre_build: echo "pre-build section for GCC compilers"
post_build: echo "post-build section for GCC compilers"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
config:
gcc@7.3.0:
pre_build: echo "pre-build section for gcc@7.3.0"
post_build: echo "post-build section for gcc@7.3.0"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
cflags: "-O3"
gcc@8.2.0:
pre_build: echo "gcc --version"
cflags: "-O3"
multi_compilers:
type: compiler
description: Select one or more compilers to run test
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc|intel|pgi|cray)"]
exclude: [intel@18]
default:
gcc:
cflags: "-fopenmp"
intel:
cflags: "-qopenmp"
pgi:
cflags: "-fopenmp"
cray:
cflags: "-h omp"
config:
gcc@7.5.0:
cflags: "-O3"
module:
load: [gcc/7.5.0]
intel@17:
module:
load: [intel/2017]
intel@18:
module:
load: [intel/2018]
pgi/18.0:
module:
load: [pgi/18.0]
craype/2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray]
load: [craype/2.6.2]
Spack Schema Examples¶
buildtest schema -n spack.schema.json --example
$ buildtest schema -n spack.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
specs_must_be_list_of_strings:
type: spack
executor: generic.local.sh
description: 'specs must be a list of strings '
spack:
root: $HOME/spack
env:
create:
name: myproject
specs:
- 1
- zlib
additionalProperties_spack_field:
type: spack
executor: generic.local.sh
description: additional Properties can't be specified in spack section
spack:
root: $HOME/spack
FOO: BAR
env:
create:
name: myproject
specs:
- zlib
invalid_type_mirror_field:
type: spack
executor: generic.local.sh
description: The mirror field must be a key value pair
spack:
root: $HOME/spack
mirror: https://caches.e4s.io
env:
create:
name: myproject
specs:
- zlib
spack_test_additionalProperties:
type: spack
executor: generic.local.sh
description: "Check for additionalProperties in test section. FOO key is not
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib']
test:
FOO: BAR
remove_tests: true
run:
specs: ['m4', 'zlib']
spack_test_run_invalid_spec:
type: spack
executor: generic.local.sh
description: "specs property requires a list of strings. "
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib']
test:
remove_tests: true
run:
specs: ['m4', 1]
results:
option: '-f'
post_cmds: |
spack find
rm -rf $SPACK_ROOT
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_create_name:
type: spack
executor: generic.local.sh
description: Create spack environment by name
spack:
root: $HOME/spack/
env:
create:
name: myproject
specs:
- m4
- zlib
install:
options: ''
env_activate:
type: spack
executor: generic.local.sh
description: Activate spack environment by name
spack:
root: $HOME/spack/
env:
activate:
name: myproject
specs:
- m4
- zlib
install:
options: ''
env_create_directory:
type: spack
executor: generic.local.sh
description: Create spack environment by directory
spack:
root: $HOME/spack/
env:
create:
dir: $HOME/spack-env/myproject
specs:
- 'm4'
- 'zlib@1.2.11'
install:
options: '--cache-only'
env_create_from_manifest:
type: spack
executor: generic.local.sh
description: Create spack enviromment from manifest file
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
env_concretized_install:
type: spack
description: run 'spack concretize -f' in an environment and install specs
executor: generic.local.sh
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
concretize: true
install:
options: '--cache-only'
env_mirror:
type: spack
executor: generic.local.sh
description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in e
spack:
root: $HOME/spack/
env:
mirror:
h5: /path/to/mirror
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
install_without_env:
type: spack
executor: generic.local.sh
description: install specs without environment
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
pre_post_cmd_spack_install:
type: spack
executor: generic.local.sh
description: run commands before and after spack using pre_cmds and post_cmd
pre_cmds: |
cd $HOME
git clone https://github.com/spack/spack
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
post_cmds: |
spack find
remove_spack_environment:
type: spack
executor: generic.local.sh
description: remove spack environment explicitly before creating environment
spack:
root: $HOME/spack
env:
rm:
name: m4
create:
name: m4
activate:
name: m4
specs:
- 'm4'
remove_spack_environment_automatically:
type: spack
executor: generic.local.sh
description: remove spack environment automatically
spack:
root: $HOME/spack
env:
create:
remove_environment: true
name: m4
activate:
name: m4
specs:
- 'm4'
post_cmds: |
spack find
rm -rf $SPACK_ROOT
sbatch_field_with_spack:
type: spack
executor: generic.local.sh
description: Specify sbatch field with spack schema
sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"]
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
skip_test_in_spack:
skip: True
type: spack
executor: generic.local.sh
description: This test will be skipped
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
var_declaration_in_spack:
type: spack
executor: generic.local.sh
description: Define variables and environment variables in spack
vars:
FOO: "BAR"
env:
SPACK_GNUPGHOME: "$HOME/.gnupg"
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
spack_sbatch_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "sbatch directives can be defined in spack schema"
tags: [spack]
executors:
generic.local.sh:
sbatch: ["-N 1", "-t 20"]
generic.local.bash:
sbatch: ["-N 8", "-t 10"]
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true
vars_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "variable declaration with multiple executors"
tags: [spack]
executors:
generic.local.sh:
vars:
FOO: BAR
generic.local.bash:
vars:
HELLO: WORLD
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true