Buildtest Schemas
buildtest uses JSON Schema for validating buildspecs and buildtest configuration file.
In this section, we will show how you can query the json schema and its example files using the command line tool buildtest schema.
Schema Naming Convention
All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.
CLI for buildtest schemas (buildtest schema)
You can use buildtest schema command to see the list of schemas
supported by buildtest. The schema files are denoted by .schema.json file extension.
buildtest schema
$ buildtest schema
global.schema.json
definitions.schema.json
settings.schema.json
spack.schema.json
script.schema.json
Shown below is the command usage of buildtest schema
buildtest schema --help
$ buildtest schema --help
usage: buildtest [options] [COMMANDS] schema [-h] [-e] [-j] [-n Schema Name]
optional arguments:
-h, --help show this help message and exit
-e, --example Show schema examples
-j, --json Display json schema file
-n Schema Name, --name Schema Name
show schema by name (e.g., script)
You must use the --name option to select a schema, for instance if you want to view the JSON Schema for
script.schema.json you can run the following:
buildtest schema --name script.schema.json --json
Schema Files
Definition Schema
This schema is used for declaring definitions that need to be
reused in multiple schemas. We use $ref keyword to reference definitions from this file.
definitions.schema.json
{
"$id": "definitions.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "JSON Schema Definitions File",
"description": "This file is used for declaring definitions that are referenced from other schemas",
"definitions": {
"list_of_strings": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"string_or_list": {
"oneOf": [
{
"type": "string"
},
{
"$ref": "#/definitions/list_of_strings"
}
]
},
"list_of_ints": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "integer"
}
},
"list_of_positive_integers": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "integer",
"minimum": 1
}
},
"int_or_list": {
"oneOf": [
{
"type": "integer"
},
{
"$ref": "#/definitions/list_of_ints"
}
]
},
"file": {
"type": "string",
"description": "Specify a file name to read"
},
"file_regex": {
"type": "array",
"description": "Specify a list of regular expressions to match files in the current working directory. The regular expression is matched using ``re.search`` python module. ",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"file",
"exp"
],
"properties": {
"file": {
"$ref": "#/definitions/file"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run on the selected file name"
},
"item": {
"type": "integer",
"minimum": 0,
"description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
},
"re": {
"type": "string",
"description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
"enum": [
"re.search",
"re.match",
"re.fullmatch"
]
}
}
}
},
"file_regex_in_metrics": {
"type": "object",
"additionalProperties": false,
"description": "Specify a regular expressions on a filepath used for assigning value to metrics. The regular expression is matched using ``re.search`` python module. ",
"required": [
"file",
"exp"
],
"properties": {
"additionalProperties": false,
"file": {
"$ref": "#/definitions/file"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run on the selected file name"
},
"item": {
"type": "integer",
"minimum": 0,
"description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
},
"linenum": {
"type": "integer",
"description": "Specify the line number used to extract from the file content where regex is applied"
},
"re": {
"type": "string",
"description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
"enum": [
"re.search",
"re.match",
"re.fullmatch"
]
}
}
},
"regex": {
"type": "object",
"additionalProperties": false,
"description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
"required": [
"exp",
"stream"
],
"properties": {
"stream": {
"type": "string",
"enum": [
"stdout",
"stderr"
],
"description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
},
"item": {
"type": "integer",
"minimum": 0,
"description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
},
"linenum": {
"type": "integer",
"description": "Specify the line number used to extract from the content where regex is applied"
},
"re": {
"type": "string",
"description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
"enum": [
"re.search",
"re.match",
"re.fullmatch"
]
}
}
},
"env": {
"type": "object",
"description": "One or more key value pairs for an environment (key=value)",
"minItems": 1,
"items": {
"type": "object",
"minItems": 1,
"propertyNames": {
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
}
}
},
"description": {
"type": "string",
"description": "The ``description`` field is used to document what the test is doing",
"maxLength": 80
},
"summary": {
"type": "string",
"description": "The ``summary`` field is used to document what the test is doing and can be a multi-line string"
},
"tags": {
"description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
"$ref": "#/definitions/string_or_list"
},
"skip": {
"type": "boolean",
"description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
},
"executor": {
"type": "string",
"description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
},
"needs": {
"description": "A list of test names that are dependency before runnning job",
"type": "array",
"items": [
{
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"patternProperties": {
"^.*$": {
"additionalProperties": false,
"properties": {
"state": {
"$ref": "#/definitions/state"
},
"returncode": {
"$ref": "#/definitions/returncode"
}
}
}
}
}
]
}
]
},
"metrics_field": {
"type": "object",
"additionalProperties": false,
"required": [
"type"
],
"properties": {
"type": {
"type": "string",
"description": "Specify python data-type (str, int, float) to convert metric. ",
"enum": [
"str",
"int",
"float"
]
},
"regex": {
"$ref": "#/definitions/regex"
},
"file_regex": {
"$ref": "#/definitions/file_regex_in_metrics"
}
},
"oneOf": [
{
"required": [
"regex"
]
},
{
"required": [
"file_regex"
]
}
]
},
"metrics": {
"type": "object",
"description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/metrics_field"
}
},
"state": {
"type": "string",
"description": "explicitly mark state of test regardless of status calculation",
"enum": [
"PASS",
"FAIL"
]
},
"returncode": {
"description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
"$ref": "#/definitions/int_or_list"
},
"status": {
"type": "object",
"description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
"additionalProperties": false,
"properties": {
"slurm_job_state": {
"type": "string",
"enum": [
"COMPLETED",
"FAILED",
"OUT_OF_MEMORY",
"TIMEOUT"
],
"description": "This field can be used to pass test based on Slurm Job State, if there is a match buildtest will report as ``PASS`` "
},
"pbs_job_state": {
"type": "string",
"description": "This field can be used to pass test based on PBS Job State, if there is a match buildtest will report as ``PASS`` ",
"enum": [
"H",
"S",
"F"
]
},
"lsf_job_state": {
"type": "string",
"description": "This field can be used to pass test based on LSF Job State, if there is a match buildtest will report as ``PASS`` ",
"enum": [
"DONE",
"EXIT"
]
},
"returncode": {
"$ref": "#/definitions/returncode"
},
"regex": {
"$ref": "#/definitions/regex",
"description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
},
"file_regex": {
"$ref": "#/definitions/file_regex",
"description": "Perform a regular expression search on a list of arbitrary files, if expression matches for any file buildtest will report as ``PASS``. This field is useful for checking output of a test that is not captured in stdout or stderr. "
},
"runtime": {
"type": "object",
"description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
"properties": {
"min": {
"type": "number",
"minimum": 0,
"description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
},
"max": {
"type": "number",
"minimum": 0,
"description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
}
}
},
"assert_ge": {
"description": "Perform assertion of greater and equal (>=) with reference value",
"$ref": "#/definitions/status_check_ge_gt_le_lt"
},
"assert_le": {
"description": "Perform assertion of greater and equal (>=) with reference value",
"$ref": "#/definitions/status_check_ge_gt_le_lt"
},
"assert_gt": {
"description": "Perform assertion of greater than (>) with reference value",
"$ref": "#/definitions/status_check_ge_gt_le_lt"
},
"assert_lt": {
"description": "Perform assertion of less than (<) with reference value",
"$ref": "#/definitions/status_check_ge_gt_le_lt"
},
"assert_eq": {
"description": "Perform assertion of equality (=) with reference value",
"$ref": "#/definitions/status_check_eq_ne"
},
"assert_ne": {
"description": "Perform assertion of not equal (!=) with reference value",
"$ref": "#/definitions/status_check_eq_ne"
},
"assert_range": {
"type": "object",
"description": "Perform assertion based on lower and upper bound",
"additionalProperties": false,
"required": [
"comparisons"
],
"properties": {
"mode": {
"$ref": "#/definitions/mode"
},
"comparisons": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"lower",
"upper"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"lower": {
"description": "Specify reference value for lower bound",
"type": "number"
},
"upper": {
"description": "Specify reference value for upper bound",
"type": "number"
}
}
}
}
}
},
"contains": {
"description": "Check if metric value is in a list of reference values",
"$ref": "#/definitions/status_check_contains_not_contains"
},
"not_contains": {
"description": "Check if metric value not in a list of reference values ",
"$ref": "#/definitions/status_check_contains_not_contains"
},
"is_symlink": {
"description": "Check for list of files or directory paths that are symbolic links",
"$ref": "#/definitions/list_of_strings"
},
"exists": {
"description": "Check for list of file or directory path for existences using os.path.exists",
"$ref": "#/definitions/list_of_strings"
},
"is_dir": {
"description": "Check for list of filepaths are directories",
"$ref": "#/definitions/list_of_strings"
},
"is_file": {
"description": "Check for list of filepaths are files",
"$ref": "#/definitions/list_of_strings"
},
"file_count": {
"description": "Perform assertion check by comparing file count in a directory",
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"dir",
"count"
],
"properties": {
"dir": {
"type": "string",
"description": "Directory path to check for file count"
},
"count": {
"type": "integer",
"description": "Number of files expected in directory",
"minimum": 0
},
"depth": {
"type": "integer",
"description": "Depth of directory to search for files",
"minimum": 0
},
"ext": {
"$ref": "#/definitions/string_or_list",
"description": "File extension to filter files in directory"
},
"filepattern": {
"type": "string",
"description": "Specify a regular expression when searching for files in directory"
},
"filetype": {
"type": "string",
"description": "Specify file type when searching for files in directory. It can be 'file', 'dir' or 'symlink'",
"enum": [
"file",
"dir",
"symlink"
]
},
"file_traverse_limit": {
"type": "integer",
"description": "Limit the number of files to traverse when searching for files in directory",
"minimum": 1,
"default": 10000,
"maximum": 999999
}
}
}
},
"state": {
"$ref": "#/definitions/state",
"description": "explicitly mark state of test regardless of status calculation"
},
"mode": {
"$ref": "#/definitions/mode"
},
"linecount": {
"type": "object",
"description": "Check for number of lines in stdout or stderr stream. If linecount is set, buildtest will check for number of lines in stdout or stderr stream and compare with linecount. If linecount matches buildtest will report as ``PASS``",
"additionalProperties": false,
"required": [
"stream",
"count"
],
"properties": {
"stream": {
"type": "string",
"enum": [
"stdout",
"stderr"
],
"description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if linecount matches"
},
"count": {
"type": "integer",
"description": "Specify number of lines as reference value to compare with actual output",
"minimum": 0
}
}
},
"file_linecount": {
"type": "array",
"description": "Specify a list of files to perform line count comparison by reading the file and comparing the line count with reference value",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"file",
"count"
],
"properties": {
"file": {
"$ref": "#/definitions/file"
},
"count": {
"type": "integer",
"description": "Specify number of lines as reference value to compare with actual output",
"minimum": 0
}
}
}
}
}
},
"status_check_ge_gt_le_lt": {
"type": "object",
"additionalProperties": false,
"required": [
"comparisons"
],
"properties": {
"mode": {
"$ref": "#/definitions/mode"
},
"comparisons": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "number",
"description": "Specify reference value (int,float) for comparison"
}
}
}
}
}
},
"status_check_eq_ne": {
"type": "object",
"additionalProperties": false,
"required": [
"comparisons"
],
"properties": {
"mode": {
"$ref": "#/definitions/mode"
},
"comparisons": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"description": "Specify reference value (str, int, float) for comparison",
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
}
}
}
}
}
},
"status_check_contains_not_contains": {
"type": "object",
"required": [
"comparisons"
],
"additionalProperties": false,
"properties": {
"mode": {
"$ref": "#/definitions/mode"
},
"comparisons": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"ref"
],
"properties": {
"name": {
"type": "string",
"description": "Name of metric to use for comparison"
},
"ref": {
"type": "array",
"minItems": 1,
"description": "Specify a list of reference value"
}
}
}
}
}
},
"mode": {
"description": "Determine how the status check is resolved, for instance it can be logical AND or OR",
"type": "string",
"enum": [
"or",
"and",
"OR",
"AND"
]
},
"BB": {
"$ref": "#/definitions/list_of_strings",
"description": "Create burst buffer space, this specifies #BB options in your test."
},
"DW": {
"$ref": "#/definitions/list_of_strings",
"description": "Specify Data Warp option (#DW) when using burst buffer."
},
"sbatch": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #SBATCH options in test script."
},
"bsub": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #BSUB options in test script."
},
"pbs": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #PBS directives in test script."
},
"executors": {
"type": "object",
"description": "Define executor specific configuration",
"patternProperties": {
"description": "Name of executor to override configuration",
"^.*$": {
"additionalProperties": false,
"properties": {
"env": {
"$ref": "#/definitions/env"
},
"vars": {
"$ref": "#/definitions/env"
},
"sbatch": {
"$ref": "#/definitions/list_of_strings"
},
"bsub": {
"$ref": "#/definitions/list_of_strings"
},
"pbs": {
"$ref": "#/definitions/list_of_strings"
},
"BB": {
"$ref": "#/definitions/BB"
},
"DW": {
"$ref": "#/definitions/DW"
},
"status": {
"$ref": "#/definitions/status"
},
"metrics": {
"$ref": "#/definitions/metrics"
}
}
}
}
},
"cc": {
"type": "string",
"description": "Set C compiler wrapper"
},
"fc": {
"type": "string",
"description": "Set Fortran compiler wrapper"
},
"cxx": {
"type": "string",
"description": "Set C++ compiler wrapper"
},
"cflags": {
"type": "string",
"description": "Set C compiler flags."
},
"fflags": {
"type": "string",
"description": "Set Fortran compiler flags."
},
"cxxflags": {
"type": "string",
"description": "Set C++ compiler flags."
},
"ldflags": {
"type": "string",
"description": "Set linker flags"
},
"cppflags": {
"type": "string",
"description": "Set C or C++ preprocessor flags"
},
"run": {
"type": "string",
"description": "Specify a series of commands to run."
},
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"restore": {
"description": "Load a collection name via ``module restore``",
"type": "string"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": {
"type": "string"
}
}
}
}
}
}
Settings Schema
This schema defines how buildtest configuration file is validated.
settings.schema.json
{
"$id": "settings.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "buildtest configuration schema",
"type": "object",
"required": [
"system"
],
"additionalProperties": false,
"properties": {
"system": {
"type": "object",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/system"
}
}
}
},
"definitions": {
"terse": {
"type": "boolean",
"description": "A boolean to determine whether to enable terse mode"
},
"format": {
"type": "string",
"description": "Determine the format fields to display when viewing table results"
},
"count": {
"type": "integer",
"minimum": 1,
"description": "Determine number of records to display in a table"
},
"system": {
"required": [
"executors",
"moduletool",
"hostnames",
"compilers",
"buildspecs",
"report"
],
"additionalProperties": false,
"type": "object",
"properties": {
"hostnames": {
"type": "array",
"description": "Specify a list of hostnames to check where buildtest can run for the given system record",
"items": {
"type": "string"
}
},
"description": {
"type": "string",
"description": "system description field"
},
"poolsize":
{
"type": "integer",
"minimum": 1,
"description": "Specify size of Process Pool for parallel processing using ``multiprocessing.Pool``"
},
"max_jobs": {
"type": "integer",
"minimum": 1,
"description": "Maximum number of jobs that can be run at a given time for a particular executor"
},
"testdir": {
"type": "string",
"description": "Specify full path to test directory where buildtest will write tests."
},
"logdir": {
"type": "string",
"description": "Specify location where buildtest will write log files"
},
"moduletool": {
"type": "string",
"description": "Specify modules tool used for interacting with ``module`` command. ",
"enum": [
"environment-modules",
"lmod",
"none"
]
},
"timeout": {
"type": "integer",
"minimum": 1,
"description": "Specify timeout duration in number of seconds"
},
"pager":
{
"type": "boolean",
"description": "A boolean to determine whether to enable paging when viewing buildspec cache"
},
"buildspecs":
{
"type": "object",
"description": "Specify configuration for ``buildtest buildspec`` command",
"additionalProperties": false,
"properties": {
"rebuild": {
"type": "boolean",
"description": "A boolean to determine whether to rebuild buildspec cache"
},
"count": { "$ref": "#/definitions/count" } ,
"format": {
"$ref": "#/definitions/format"
},
"terse": { "$ref": "#/definitions/terse" },
"directory": {
"type": "array",
"items": {
"type": "string"
},
"description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
}
}
},
"report":
{
"type": "object",
"additionalProperties": false,
"description": "Specify configuration for ``buildtest report`` command",
"properties": {
"terse": { "$ref": "#/definitions/terse" },
"format": { "$ref": "#/definitions/format" },
"count": { "$ref": "#/definitions/count" }
}
},
"compilers": {
"type": "object",
"description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
"additionalProperties": false,
"properties": {
"enable_prgenv": {
"type": "boolean",
"description": "Enable support for Programming Environment"
},
"modulepath": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"purge": {"type": "boolean", "description": "A boolean to determine whether to purge modules via ``module purge`` when generating compiler declaration"},
"prgenv_modules": {
"type": "object",
"additionalProperties": false,
"properties": {
"gcc": {
"type": "string",
"description": "Specify name of Programming Environment module for gcc",
"enum": [
"PrgEnv-gnu"
]
},
"intel": {
"type": "string",
"description": "Specify name of Programming Environment module for intel",
"enum": [
"PrgEnv-intel"
]
},
"cray": {
"type": "string",
"description": "Specify name of Programming Environment module for cray",
"enum": [
"PrgEnv-cray"
]
},
"nvhpc": {
"type": "string",
"description": "Specify name of Programming Environment module for nvhpc",
"enum": [
"PrgEnv-nvhpc",
"PrgEnv-nvidia"
]
}
}
},
"find": {
"type": "object",
"additionalProperties": false,
"description": "Find compilers by specifying regular expression that is applied to modulefile names",
"properties": {
"gcc": {
"type": "string",
"description": "Specify a regular expression to search for gcc compilers from your module stack"
},
"intel": {
"type": "string",
"description": "Specify a regular expression to search for intel compilers from your module stack"
},
"cray": {
"type": "string",
"description": "Specify a regular expression to search for cray compilers from your module stack"
},
"clang": {
"type": "string",
"description": "Specify a regular expression to search for clang compilers from your module stack"
},
"cuda": {
"type": "string",
"description": "Specify a regular expression to search for cuda compilers from your module stack"
},
"pgi": {
"type": "string",
"description": "Specify a regular expression to search for pgi compilers from your module stack"
},
"upcxx": {
"type": "string",
"description": "Specify a regular expression to search for upcxx compilers from your module stack"
},
"nvhpc": {
"type": "string",
"description": "Specify a regular expression to search for nvhpc compilers from your module stack"
}
}
},
"compiler": {
"type": "object",
"additionalProperties": false,
"description": "Start of compiler declaration",
"properties": {
"gcc": {
"description": "Declaration of one or more GNU compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"intel": {
"description": "Declaration of one or more Intel compilers. ",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cray": {
"description": "Declaration of one or more Cray compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"pgi": {
"description": "Declaration of one or more PGI compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"clang": {
"description": "Declaration of one or more Clang compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cuda": {
"description": "Declaration of one or more CUDA compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"upcxx": {
"description": "Declaration of one or more UPCXX compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"nvhpc": {
"description": "Declaration of one or more NVHPC compilers.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
}
}
}
}
},
"executors": {
"type": "object",
"additionalProperties": false,
"description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``. The executors are referenced in buildspec using ``executor`` field.",
"required": ["local"],
"properties": {
"defaults": {
"type": "object",
"description": "Specify default executor settings for all executors",
"additionalProperties": false,
"properties": {
"pollinterval": {
"type": "integer",
"description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
"minimum": 1,
"default": 30
},
"maxpendtime": {
"$ref": "#/definitions/maxpendtime"
},
"account": {
"$ref": "#/definitions/account"
}
}
},
"local": {
"type": "object",
"description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/local"
},
"minProperties": 1
},
"lsf": {
"type": "object",
"description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/lsf"
}
},
"slurm": {
"type": "object",
"description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/slurm"
}
},
"pbs": {
"type": "object",
"description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/pbs"
}
},
"torque": {
"type": "object",
"description": "The ``torque`` section is used for declaring PBS/Torque executors for running jobs using Torque scheduler",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/pbs"
}
},
"container": {
"type": "object",
"description": "The ``container`` section is used for declaring container executors for running jobs using container technology",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/container"
}
}
}
},
"cdash": {
"type": "object",
"description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
"required": ["url", "project", "site"],
"properties": {
"url": {
"type": "string",
"description": "Url to CDASH server"
},
"project": {
"type": "string",
"description": "Name of CDASH project"
},
"site": {
"type": "string",
"description": "Site Name reported in CDASH"
}
}
},
"paths":
{
"type": "object",
"description": "Specify paths to look up for executables",
"additionalProperties": false,
"properties": {
"slurm": {
"type": "string",
"description": "Specify path to slurm executable"
},
"lsf": {
"type": "string",
"description": "Specify path to lsf executable"
},
"pbs": {
"type": "string",
"description": "Specify path to pbs executable"
},
"torque": {
"type": "string",
"description": "Specify path to torque executable"
},
"docker": {
"type": "string",
"description": "Specify path to docker executable"
},
"singularity": {
"type": "string",
"description": "Specify path to singularity executable"
},
"podman": {
"type": "string",
"description": "Specify path to podman executable"
}
}
},
"profiles": {
"type": "object",
"description": "The profiles section is used for declaring one or more profiles that can be used to run ``buildtest build`` that are captured as command options ",
"propertyNames": {
"pattern": "^[A-Za-z0-9_.-]+$"
},
"additionalProperties": {
"$ref": "#/definitions/profiles"
}
}
}
},
"cc": {
"description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
"type": "string"
},
"cxx": {
"type": "string",
"description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"fc": {
"type": "string",
"description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"compiler_section": {
"description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
"type": "object",
"additionalProperties": false,
"required": [ "cc", "cxx", "fc" ],
"properties": {
"cc": { "$ref": "#/definitions/cc" },
"cxx": { "$ref": "#/definitions/cxx" },
"fc": { "$ref": "#/definitions/fc" },
"module": { "$ref": "#/definitions/module" }
}
},
"unique_string_array": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"disable": {"type": "boolean", "description": "Disable an executor"},
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": { "type": "string" }
},
"restore": {
"description": "Load a collection name via ``module restore``",
"type": "string"
}
}
},
"script": {
"type": "array",
"additionalProperties": false,
"items": { "type": "string" }
},
"maxpendtime": {
"type": "integer",
"description": "Cancel job if it is still pending in queue beyond maxpendtime",
"minimum": 1,
"default": 86400
},
"account": {
"type": "string",
"description": "Specify Job Account for charging resources"
},
"local": {
"type": "object",
"description": "An instance object of local executor",
"additionalProperties": false,
"required": [ "shell" ],
"properties": {
"description": { "$ref": "#/definitions/description" },
"shell": {
"type": "string",
"description": "Specify the shell launcher you want to use when running tests locally"
},
"before_script": { "#ref": "#/definitions/script" },
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" }
}
},
"slurm": {
"type": "object",
"additionalProperties": false,
"description": "An instance object of slurm executor",
"properties": {
"description": { "$ref": "#/definitions/description" },
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
},
"cluster": {
"type": "string",
"description": "Specify the slurm cluster you want to use ``-M <cluster>``"
},
"partition": {
"type": "string",
"description": "Specify the slurm partition you want to use ``-p <partition>``"
},
"qos": {
"type": "string",
"description": "Specify the slurm qos you want to use ``-q <qos>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" },
"disable_check": {
"type": "boolean",
"description": "Disable check for partition or qos "
}
}
},
"lsf": {
"type": "object",
"description": "An instance object of lsf executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" },
"disable_check": {
"type": "boolean",
"description": "Disable check for partition or qos "
}
}
},
"pbs": {
"type": "object",
"description": "An instance object of pbs executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": { "$ref": "#/definitions/description" },
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"maxpendtime": {
"description": "overrides default ``maxpendtime`` value",
"$ref": "#/definitions/maxpendtime"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"disable": {"$ref": "#/definitions/disable"},
"module": { "$ref": "#/definitions/module" },
"disable_check": {
"type": "boolean",
"description": "Disable check for partition or qos "
}
}
},
"container": {
"type": "object",
"additionalProperties": false,
"properties": {
"description": { "$ref": "#/definitions/description" },
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"platform": {
"type": "string",
"description": "Specify container platform to use for container. ",
"enum": [
"docker",
"singularity",
"podman"
]
},
"image": {
"type": "string",
"description": "Specify container image to use for container. "
},
"mounts": {
"type": "string",
"description": "Specify a list of directory paths to bind mount into the container"
},
"options": {
"type": "string",
"description": "Specify a list of options to pass to container runtime. "
},
"disable": {"$ref": "#/definitions/disable"}
}
},
"profiles":
{
"type": "object",
"properties": {
"buildspecs": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"exclude-buildspecs": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"tags": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"exclude-tags": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"executors": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"filter": {
"type": "object",
"additionalProperties": false,
"properties": {
"type": {
"type": "array",
"description": "Specify the `type` field to determine which schema to use during filtering",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string",
"enum": ["script", "spack"]
}
},
"tags": {
"description": "Filter by tags",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"maintainers": {
"description": "Filter by maintainers",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
}
}
},
"module": {"type": "string"},
"unload-modules": {"type": "string"},
"module-purge": {"type": "boolean"},
"validate": {"type": "boolean", "description": "Validate given buildspecs and buildtest will stop after parse stage"},
"display": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
"dry-run": {"type": "boolean", "description": "Show a list of tests that will potentially be run"},
"rebuild": {"type": "integer", "minimum": 1, "maximum": 50, "description": "Specify number of tests to rebuild"},
"limit": {"type": "integer", "minimum": 1, "description": "Limit number of tests to build"},
"account": {"$ref": "#/definitions/account"},
"maxpendtime": { "$ref": "#/definitions/maxpendtime" },
"pollinterval": {"type": "integer", "minimum": 1, "description": "Specify poll interval in seconds"},
"procs": {"$ref": "definitions.schema.json#/definitions/list_of_positive_integers"},
"nodes": {"$ref": "definitions.schema.json#/definitions/list_of_positive_integers"},
"testdir": {"type": "string", "description": "Specify test directory where buildtest will write tests."},
"timeout": {"type": "integer", "minimum": 1},
"executor-type": {
"type": "string",
"description": "Specify the `executor-type` field used by ``--executor-type`` option which determines if test will run by local or batch executor",
"enum": ["local", "batch"]
},
"remove-stagedir": {"type": "boolean", "description": "Remove stage directory after test completes"},
"max-jobs": {"type": "integer", "minimum": 1, "description": "Maximum number of jobs that can be run at a given time for a particular executor"},
"strict": {"type": "boolean", "description": "Enable strict mode for buildtest"}
}
},
"description": {
"type": "string",
"description": "description field for documenting your executor"
}
}
}
Global Schema
This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema
global.schema.json
{
"$id": "global.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "global schema",
"description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
"type": "object",
"required": ["buildspecs"],
"additionalProperties": false,
"properties": {
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"maintainers": {
"type": "array",
"description": "One or more maintainers or aliases",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"buildspecs": {
"type": "object",
"description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
"propertyNames": {
"pattern": "^[A-Za-z_.-][A-Za-z0-9_.-]*$",
"maxLength": 48
}
}
}
}
Script Schema
This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when
type: script is specified. For more details on script schema see Script Schema.
script.schema.json
{
"$id": "script.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "script schema version",
"description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
"type": "object",
"required": [
"type",
"run",
"executor"
],
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^script$",
"description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
},
"description": {
"$ref": "definitions.schema.json#/definitions/description"
},
"summary": {
"$ref": "definitions.schema.json#/definitions/summary"
},
"sbatch": {
"$ref": "definitions.schema.json#/definitions/sbatch"
},
"bsub": {
"$ref": "definitions.schema.json#/definitions/bsub"
},
"pbs": {
"$ref": "definitions.schema.json#/definitions/pbs"
},
"BB": {
"$ref": "definitions.schema.json#/definitions/BB"
},
"DW": {
"$ref": "definitions.schema.json#/definitions/DW"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"executor": {
"$ref": "definitions.schema.json#/definitions/executor"
},
"needs": {
"$ref": "definitions.schema.json#/definitions/needs"
},
"shell": {
"type": "string",
"description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed"
},
"shebang": {
"type": "string",
"description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
},
"post_run": {
"$ref": "definitions.schema.json#/definitions/run"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"tags": {
"$ref": "definitions.schema.json#/definitions/tags"
},
"metrics": {
"$ref": "definitions.schema.json#/definitions/metrics"
},
"executors": {
"$ref": "definitions.schema.json#/definitions/executors"
},
"compilers": {
"type": "object",
"required": [
"name"
],
"additionalProperties": false,
"properties": {
"name": {
"description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"exclude": {
"description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"default": {
"type": "object",
"additionalProperties": false,
"properties": {
"gcc": {
"$ref": "#/definitions/default_compiler_config"
},
"intel": {
"$ref": "#/definitions/default_compiler_config"
},
"pgi": {
"$ref": "#/definitions/default_compiler_config"
},
"cray": {
"$ref": "#/definitions/default_compiler_config"
},
"clang": {
"$ref": "#/definitions/default_compiler_config"
},
"cuda": {
"$ref": "#/definitions/default_compiler_config"
},
"upcxx": {
"$ref": "#/definitions/default_compiler_config"
},
"nvhpc": {
"$ref": "#/definitions/default_compiler_config"
}
}
},
"config": {
"type": "object",
"description": "Specify compiler configuration based on named compilers.",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/compiler_declaration"
}
}
}
}
},
"container": {
"type": "object",
"required": [
"image",
"platform"
],
"properties": {
"platform": {
"type": "string",
"description": "Specify a container platform to use when running test. This is used for running commands inside container.",
"enum": [
"docker",
"podman",
"singularity"
]
},
"image": {
"type": "string",
"description": "Specify a container image to use when running test. This is used for running commands inside container."
},
"mounts": {
"type": "string",
"description": "Specify a list of directory paths to bind mount into the container"
},
"options": {
"type": "string",
"description": "Specify a list of options to pass to container runtime. "
},
"command": {
"type": "string",
"description": "Specify a list of commands to run inside container. This is used for running commands inside container."
}
}
}
},
"definitions": {
"compiler_declaration": {
"type": "object",
"description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
"additionalProperties": false,
"properties": {
"cc": {
"$ref": "definitions.schema.json#/definitions/cc"
},
"fc": {
"$ref": "definitions.schema.json#/definitions/fc"
},
"cxx": {
"$ref": "definitions.schema.json#/definitions/cxx"
},
"cflags": {
"$ref": "definitions.schema.json#/definitions/cflags"
},
"fflags": {
"$ref": "definitions.schema.json#/definitions/fflags"
},
"cxxflags": {
"$ref": "definitions.schema.json#/definitions/cxxflags"
},
"ldflags": {
"$ref": "definitions.schema.json#/definitions/ldflags"
},
"cppflags": {
"$ref": "definitions.schema.json#/definitions/cppflags"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
},
"module": {
"$ref": "definitions.schema.json#/definitions/module"
}
}
},
"default_compiler_config": {
"type": "object",
"description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
"properties": {
"cc": {
"$ref": "definitions.schema.json#/definitions/cc"
},
"fc": {
"$ref": "definitions.schema.json#/definitions/fc"
},
"cxx": {
"$ref": "definitions.schema.json#/definitions/cxx"
},
"cflags": {
"$ref": "definitions.schema.json#/definitions/cflags"
},
"fflags": {
"$ref": "definitions.schema.json#/definitions/fflags"
},
"cxxflags": {
"$ref": "definitions.schema.json#/definitions/cxxflags"
},
"ldflags": {
"$ref": "definitions.schema.json#/definitions/ldflags"
},
"cppflags": {
"$ref": "definitions.schema.json#/definitions/cppflags"
},
"env": {
"$ref": "definitions.schema.json#/definitions/env"
},
"vars": {
"$ref": "definitions.schema.json#/definitions/env"
},
"status": {
"$ref": "definitions.schema.json#/definitions/status"
},
"run": {
"$ref": "definitions.schema.json#/definitions/run"
}
}
}
}
}
Spack Schema
This schema is used for writing tests with spack package manager using type: spack field. For more details
see Buildtest Spack Integration.
spack.schema.json
{
"$id": "spack.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "spack schema version",
"description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
"type": "object",
"required": [
"type",
"executor",
"spack"
],
"additionalProperties": false,
"definitions": {
"env": {
"additionalProperties": false,
"type": "object",
"description": "Used for managing spack environment using ``spack env`` command. ",
"properties": {
"create": {
"additionalProperties": false,
"description": "Create a spack environment via ``spack env create``",
"type": "object",
"properties": {
"remove_environment": {
"type": "boolean",
"description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
"default": false
},
"name": {
"type": "string",
"description": "Name of spack environment to create"
},
"manifest": {
"type": "string",
"description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
},
"options": {
"type": "string",
"description": "Pass options to ``spack env create`` command"
},
"dir": {
"type": "string",
"description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
}
}
},
"deactivate": {
"type": "boolean",
"description": "Deactivate a spack environment prior to activating it via ``spack env deactivate``"
},
"activate": {
"additionalProperties": false,
"type": "object",
"description": "Activate a spack environment via ``spack env activate``",
"properties": {
"name": {
"type": "string",
"description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
},
"options": {
"type": "string",
"description": "Pass options to ``spack env activate`` command"
},
"dir": {
"type": "string",
"description": "Activate spack environment from directory."
}
}
},
"rm": {
"additionalProperties": false,
"description": "Remove an existing spack environment via ``spack env rm``.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
}
}
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror in spack environment by running ``spack mirror add``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
},
"concretize": {
"type": "boolean",
"description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
}
}
},
"install": {
"description": "Install spack packages using ``spack install`` command",
"type": "object",
"properties": {
"options": {
"type": "string",
"description": "Pass options to ``spack install`` command"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to install using ``spack install`` command"
}
}
},
"load": {
"description": "load spack packages using ``spack load`` command",
"type": "object",
"properties": {
"options": {
"type": "string",
"description": "Pass options to ``spack load`` command"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to install using ``spack load`` command"
}
}
},
"test": {
"type": "object",
"additionalProperties": false,
"required": [
"run",
"results"
],
"properties": {
"remove_tests": {
"type": "boolean",
"description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
},
"run": {
"description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
"type": "object",
"required": [
"specs"
],
"additionalProperties": false,
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test run``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to run tests by running ``spack test run <specs>``."
}
}
},
"results": {
"type": "object",
"description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
"additionalProperties": false,
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test results``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Report result by spec name by running ``spack test run -- <specs>``."
}
}
}
}
}
},
"properties": {
"type": {
"type": "string",
"pattern": "^spack$",
"description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
},
"description": { "$ref": "definitions.schema.json#/definitions/description" },
"summary": { "$ref": "definitions.schema.json#/definitions/summary" },
"executor": { "$ref": "definitions.schema.json#/definitions/executor" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"skip": { "$ref": "definitions.schema.json#/definitions/skip" },
"tags": { "$ref": "definitions.schema.json#/definitions/tags" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
"executors": { "$ref": "definitions.schema.json#/definitions/executors" },
"pre_cmds": {
"type": "string",
"description": "Shell commands run before spack"
},
"post_cmds": {
"type": "string",
"description": "Shell commands run after spack"
},
"post_run": { "$ref": "definitions.schema.json#/definitions/run" },
"needs": {
"$ref": "definitions.schema.json#/definitions/needs"
},
"spack": {
"type": "object",
"description": "Entry point to spack configuration",
"additionalProperties": false,
"properties": {
"root": {
"type": "string",
"description": "Specify location for root of spack directory"
},
"compiler_find": {
"type": "boolean",
"description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror by running ``spack mirror add``"
},
"env": {
"$ref": "#/definitions/env",
"description": "Manage spack environments via ``spack env`` command"
},
"install": {
"$ref": "#/definitions/install",
"description": "Install spack packages by running ``spack install``. "
},
"load": {
"$ref": "#/definitions/load",
"description": "load spack packages by running ``spack load`` "
},
"verify_spack": {
"type": "boolean",
"description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
"default": true
},
"test": {
"$ref": "#/definitions/test",
"description": "Entry point to ``spack test``"
}
}
}
}
}
Schema Examples
The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.
In order to view an example you can run:
buildtest schema -n <schema> --example
Settings Schema Examples
buildtest schema -n settings.schema.json --example
$ buildtest schema -n settings.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: N/A
executors:
local:
# the line below is invalid due to pattern property
==bash:
description: submit jobs on local machine
shell: bash -v
slurm:
# the line below is invalid due to pattern property
$haswell:
launcher: sbatch
options: ["-p haswell", "-t 00:10"]
lsf:
<>batch:
launcher: bsub
queue: batch
options: ["-q batch", "-t 00:10"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: none
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
paths:
pbs: /usr/bin
executors:
defaults:
pollinterval: 10
maxpendtime: 30
local:
bash:
description: submit jobs via bash shell
shell: bash
pbs:
workq:
queue: workq
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
logdir: $BUILDTEST_ROOT/logs
testdir: $BUILDTEST_ROOT/tests
moduletool: none
poolsize: 1
max_jobs: 1
pager: False
cdash:
url: https://my.cdash.org
project: buildtest
site: laptop
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
executors:
local:
bash:
description: submit jobs on local machine using bash shell
shell: bash
module:
purge: True
restore: "Default"
load: ["gcc"]
sh:
description: submit jobs on local machine using sh shell
shell: sh
before_script: |
date
echo "these commands will be run"
csh:
description: submit jobs on local machine using csh shell
shell: csh -x
tcsh:
description: submit jobs on local machine using tcsh shell
shell: /bin/tcsh
zsh:
description: submit jobs on local machine using zsh shell
shell: /bin/zsh
compilers:
find:
gcc: "^(gcc|GCC|PrgEnv-gnu)"
intel: "^(intel|Intel|PrgEnv-intel)"
cray: "^(cray|PrgEnv-cray)"
clang: "^(clang|Clang)"
cuda: "^(cuda|CUDA)"
pgi: "^(pgi|PGI|PrgEnv-pgi)"
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
gcc@7.2.0:
cc: 'cc'
cxx: 'cxx'
fc: 'fc'
module:
load:
- gcc/7.2.0
intel:
intel@2019:
cc: 'icc'
cxx: 'icpc'
fc: 'ifort'
module:
purge: True
load:
- gcc/7.2.0
- intel/2019
cray:
craype@2.6.2:
cc: 'cc'
cxx: 'CC'
fc: 'fc'
module:
load: [craype/2.6.2]
swap: [PrgEnv-gnu, PrgEnv-cray]
clang:
clang@12.0.0:
cc: 'clang'
cxx: 'clang++'
fc: 'None'
module:
load: [clang/12.0]
cuda:
cuda@11.0:
cc: 'nvcc'
cxx: 'nvcc'
fc: 'None'
module:
load: [cuda/11.0]
pgi:
pgi@18.0:
cc: 'pgcc'
cxx: 'pgc++'
fc: 'pgfortran'
module:
load: [pgi/18.0]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: none
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
paths:
pbs: /usr/bin
torque: /usr/bin
lsf: /usr/bin
slurm: /usr/bin
executors:
local:
bash:
description: submit jobs on local machine
shell: bash -v
slurm:
haswell:
partition: haswell
options: ["-p haswell", "-t 00:10"]
lsf:
batch:
queue: batch
options: ["-q batch", "-t 00:10"]
pbs:
dev:
queue: development
options: ["-l walltime 00:10:00"]
torque:
dev:
queue: development
options: ["-l walltime 00:10:00"]
container:
ubuntu:
image: ubuntu:20.04
platform: docker
mounts: "/tmp:/tmp"
options: "--rm"
description: submit jobs on ubuntu container
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: lmod
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
paths:
lsf: /usr/bin
executors:
defaults:
pollinterval: 10
maxpendtime: 45
local:
bash:
description: submit jobs via bash shell
shell: bash
lsf:
batch:
description: "LSF Executor name 'batch' that submits jobs to 'batch' q
queue: batch
account: developer
options: ["-W 20"]
before_script: |
time
echo "commands run before job"
test:
description: "LSF Executor name 'test' that submits jobs to 'test' que
queue: test
account: qa
options: ["-W 20"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: lmod
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
paths:
slurm: /usr/bin
testdir: /tmp/buildtest
executors:
defaults:
pollinterval: 20
maxpendtime: 30
account: admin
local:
bash:
description: submit jobs via bash shell
shell: bash
slurm:
normal:
options: ["-C haswell"]
qos: normal
cluster: cori
before_script: |
time
echo "commands run before job"
debug:
options: ["-C cpu -t 5"]
partition: debug
cluster: perlmutter
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
system:
generic:
hostnames: ['.*']
moduletool: none
poolsize: 1
buildspecs:
# whether to rebuild cache file automatically when running `buildtest buil
rebuild: False
# limit number of records to display when running `buildtest buildspec fin
count: 15
# format fields to display when running `buildtest buildspec find`, By def
format: "name,description"
# enable terse mode
terse: False
report:
count: 25
#enable terse mode for report
terse: False
format: "name,id,state,runtime,returncode"
paths:
torque: /usr/bin
executors:
defaults:
pollinterval: 10
maxpendtime: 30
local:
bash:
description: submit jobs via bash shell
shell: bash
torque:
lbl-cluster:
queue: lbl-cluster
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
Global Schema Examples
buildtest schema -n global.schema.json --example
$ buildtest schema -n global.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
# invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$"
(badname:
type: script
run: "ping login 1"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
# this test fails because it exceeds 32 character length for test name
buildspecs:
_________this_test_exceeds_character_length______________:
type: script
run: hostname
executor: generic.local.bash
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
hostname:
type: script
run: "hostname"
maintainers: [shahzebsiddiqui, shahzebsiddiqui]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
# wrong type for maintainers key, expects a string
maintainers: 1
buildspecs:
hostname:
type: script
run: "hostname"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
# testing all caps
ABCDEFGHIJKLMNOPQRSTUVWXYZ:
type: script
run: "hostname"
# testing all lowercase letters
abcdefghijklmnopqrstuvwxyz:
type: script
run: "hostname"
# testing '_' in beginning followed by all numbers
_0123456789:
type: script
run: "hostname"
# testing '_' in combination with caps, lowercase and numbers
_ABCDEFabcdef0123456789:
type: script
run: "hostname"
# testing '_' at end of word
abcdefghijklmnopqrstuvwxyz_:
type: script
run: "hostname"
# testing '.' in beginning of word
.helloworld:
type: script
run: hostname
# testing '.' in middle of word
hello.world:
type: script
run: hostname
# testing '.' at end of word
helloworld.:
type: script
run: hostname
# testing '-' in middle of word
hello-world:
type: script
run: hostname
# testing '-' at end of word
helloworld-:
type: script
run: hostname
Script Schema Examples
buildtest schema -n script.schema.json --example
$ buildtest schema -n script.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
missing_run_key:
type: script
executor: generic.local.bash
description: invalid key name roon, missing run key
cmd: |
systemctl is-active slurmd
systemctl is-enabled slurmd | grep enabled
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
empty_tags:
type: script
executor: generic.local.bash
description: tag list can't be empty, requires one item.
tags: []
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
linecount_stream_check:
type: script
executor: generic.local.bash
description: "This test will fail because stream is not valid. It must be (s
run: |
for i in {1..10}; do
echo $i
done
status:
linecount:
stream: foo
count: 10
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_env_type:
type: script
executor: generic.local.bash
description: env key should be a dictionary
env:
- FOO=BAR
run: echo $FOO
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
non_int_returncodes_list:
type: script
executor: generic.local.bash
description: The returncode must be a list of integers and no numbers
run: exit 1
status:
returncode: [1, 2.230]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_runtime_min:
type: script
executor: generic.local.sh
description: "Invalid type for min property in runtime. Must be an integer o
run: sleep 2
status:
runtime:
min: "1"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_skip_value:
type: script
executor: generic.local.bash
description: invalid value for skip, must be boolean
skip: 1
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_linecount_negative_value:
type: script
executor: generic.local.bash
description: "Performing file count with a negative count value will result
run: |
> empty.txt
status:
file_linecount:
- file: empty.txt
count: -1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_shell_type:
type: script
executor: generic.local.bash
description: invalid shell type must be a string
shell: ["/bin/bash"]
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_returncode_type:
type: script
executor: generic.local.bash
description: This test fails because of invalid return code type
run: hostname
status:
returncode: ["1"]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_value_for_mode:
type: script
executor: generic.local.bash
description: The status mode must be 'any' or 'all'
run: exit 0
status:
returncode: 0
mode: '1'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_re_value:
type: script
executor: generic.local.bash
description: The "re" value is invalid
run: echo "world"
status:
regex:
stream: stdout
exp: "world$"
re: "search"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_tags_value:
type: script
executor: generic.local.bash
description: invalid tag value must be all string items
tags: ["network", 400]
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
runtime_min_must_exceed_0:
type: script
executor: generic.local.sh
description: "The runtime must exceed 0"
run: sleep 2
status:
runtime:
min: -1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_test_name_&!@#$%:
type: script
executor: generic.local.bash
description: "invalid test name"
run: sleep 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_regex_stream:
type: script
executor: generic.local.bash
description: This test fails because of invalid regex stream
run: hostname
status:
regex:
stream: file
exp: "world$"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_property_filename:
type: script
executor: generic.local.bash
description: Invalid property name filename under file_regex
run: echo "Hello" > hello.txt
status:
file_regex:
- filename: hello.txt
exp: '^(Hello)$'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
empty_returncode_list:
type: script
executor: generic.local.bash
description: An empty returncode list will cause an error
run: hostname
status:
returncode: []
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_type_shell_shebang:
type: script
executor: generic.local.bash
description: invalid type for shell shebang, must be a string
shebang: ["#!/bin/bash"]
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_summary:
type: script
executor: generic.local.bash
description: summary field must be a string
summary: 1
run: sleep 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_slurm_job_state:
type: script
executor: generic.local.sh
description: invalid value for slurm_job_state, should raise error with enum
sbatch:
- "-n 2"
- "-q normal"
- "-t 10"
run: hostname
status:
slurm_job_state: "FINISH"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
missing_regex_exp:
type: script
executor: generic.local.bash
description: This test fails because of missing key 'exp' in regex
run: hostname
status:
regex:
stream: stdout
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_description:
type: script
executor: generic.local.bash
description:
- "Multi Line description"
- "is not accepted"
run: sleep 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_exists_failure:
type: script
executor: generic.local.bash
description: this test will fail validation, because item must be a string
run: mkdir -p 1
status:
exists: [1]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
non_int_returncodes:
type: script
executor: generic.local.bash
description: The returncode must be an int and not a number
run: exit 1
status:
returncode: 1.01
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
executors_invalid_var_type:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Invalid type field for 'vars'
tags: [tutorials]
run: echo $FOO
executors:
generic.local.bash:
vars: ["FOO=BAR"]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_container_platform:
type: script
executor: generic.local.bash
description: invalid container platform
container:
platform: "containerd"
image: hello-world
run: echo 'Test Complete!'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_metrics_additional_property:
type: script
executor: generic.local.bash
description: Test for additional property for metrics property
vars:
FOO: BAR
run: echo $FOO
metrics:
foo:
variable: FOO
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
regex_additionalProperties_test:
type: script
executor: generic.local.bash
description: Testing for additional properties in regex field
run: hostname
status:
regex:
stream: stdout
exp: "world$"
X: 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
additionalProperties_test:
type: script
executor: generic.local.bash
description: additional properties are not allowed so any invalid key/value
FOO: BAR
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
executors_additionalProperties:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Testing for additional properties in 'executors'
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
FOO: BAR
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
generic.local.zsh:
sbatch: ["-n 16", "-N 2", "-t 120"]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
testing_additionalProperties_file_regex:
type: script
executor: generic.local.bash
description: Testing for additional properties in file_regex
run: |
echo "Hello" > hello.txt
status:
file_regex:
- file: hello.txt
exp: '^(Hello)$'
FOO: BAR
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
additionalProperties_status:
type: script
executor: generic.local.bash
description: test additional properties in status object. This is not allowe
sbatch: ["-n 2", "-q normal", "-t 10"]
run: hostname
status:
slurm_job_state: "COMPLETED"
FOO: BAR
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_additionalProperties_check:
type: script
executor: generic.local.bash
description: Testing for additional properties in file_count
run: hostname
status:
file_count:
- dir: /tmp
count: 0
FOO: BAR
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
missing_regex_in_metrics:
type: script
executor: generic.local.bash
description: Test for missing regex in metrics property, which should result
run: echo "hello"
metrics:
foo:
type: str
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
test_regex_and_file_regex_in_metrics:
type: script
executor: generic.local.bash
description: Test for regex and file_regex in metrics. Only one is allowed a
run: echo "hello" > file.txt
metrics:
foo:
type: str
file_regex:
file: file.txt
exp: "hello"
regex:
stream: stdout
exp: "hello"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_metrics_type:
type: script
executor: generic.local.bash
description: metrics property is an object, testing for type
vars:
FOO: BAR
run: echo $FOO
metrics: FOO
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
metric_type_property_test:
type: script
executor: generic.local.bash
description: metrics 'type' must be a 'str', 'float', 'int'
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: tuple
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_metrics_name:
type: script
executor: generic.local.bash
description: "Metrics name does not follow pattern"
run: echo "hello" > file.txt
metrics:
(foo-bar:
type: str
regex:
stream: stdout
exp: "BAR"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_filetype_invalid_value:
type: script
executor: generic.local.bash
description: Invalid value for filetype, it must be 'file', 'dir' or 'symlin
run: mkdir foo
status:
file_count:
- dir: foo
count: -1
filetype: 'filename'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_property_depth_failure:
type: script
executor: generic.local.bash
description: The depth property in file_count must be greater than 0
run: hostname
status:
file_count:
- dir: /tmp
count: 0
depth: -1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_property_count_failure:
type: script
executor: generic.local.bash
description: The count property in file_count must be greater than 0
run: hostname
status:
file_count:
- dir: /tmp
count: -1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_minimum_check_on_count:
type: script
executor: generic.local.bash
description: The count property must be 0 or greater
run: mkdir foo
status:
file_count:
- dir: foo
count: -1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_required_property_check:
type: script
executor: generic.local.bash
description: The dir property in file_count is required
run: hostname
status:
file_count:
- dir: /tmp
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
minimum_check_file_traverse_limit:
type: script
executor: generic.local.bash
description: The minimum value for file_traverse_limit is 1
run: mkdir foo
status:
file_count:
- dir: foo
count: 1
file_traverse_limit: 0
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_lt_example:
type: script
executor: generic.local.bash
description: Run stream test with metrics example using assert less than
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_lt:
comparisons:
- name: copy
ref: 5000
- name: scale
ref: 5500
- name: add
ref: 6000
- name: triad
ref: 6500
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
container_commands_ubuntu:
type: script
executor: generic.local.bash
description: run arbitrary linux commands in ubuntu container
container:
platform: "docker"
image: ubuntu:latest
command: bash -c "cat /etc/os-release"
run: |
ls -l /etc/os-release || true
container_options:
type: script
executor: generic.local.bash
description: run arbitrary linux commands in ubuntu container
container:
platform: "docker"
image: ubuntu:latest
options: --hostname myhostname
command: hostname
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_linecount:
type: script
executor: generic.local.bash
description: "Perform linecount comparison on files"
run: |
for i in {1..10}; do
echo $i >> count.txt
done
echo "hello world" > hello.txt
status:
file_linecount:
- file: count.txt
count: 10
- file: hello.txt
count: 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
metric_regex_example:
executor: generic.local.bash
type: script
description: capture result metric from output
run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
tags: tutorials
metrics:
hpcg_rating_stream:
type: float
regex:
exp: '(\d+\.\d+)$'
stream: stdout
hpcg_state_stream:
type: str
regex:
exp: '(VALID)'
stream: stdout
metric_file_regex:
executor: generic.local.bash
type: script
description: capture result metric from file path
run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" > hpcg.txt
tags: tutorials
metrics:
hpcg_rating_file:
type: float
file_regex:
exp: '(\d+\.\d+)$'
file: hpcg.txt
hpcg_state_file:
type: str
file_regex:
exp: '(VALID)'
file: hpcg.txt
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
status_logical_and:
type: script
executor: 'generic.local.bash'
description: 'Using logical AND to check status'
run: |
echo "This is a test"
exit 1
status:
mode: and
returncode: 1
regex:
stream: stdout
exp: 'This is a test'
status_logical_or:
type: script
executor: 'generic.local.bash'
description: 'Using logical OR to check status'
run: |
echo "This is a test"
exit 1
status:
mode: or
returncode: 0
regex:
stream: stdout
exp: 'This is a test'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_traverse_limit:
type: script
executor: generic.local.bash
description: Use of file_traverse_limit to limit number of files searched in
run: |
mkdir foo
touch foo/{1..99}.txt
status:
file_count:
- dir: foo
count: 50
file_traverse_limit: 50
- dir: foo
count: 10
file_traverse_limit: 20
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
bash_login_shebang:
type: script
executor: generic.local.bash
shebang: "#!/bin/bash -l"
description: customize shebang line with bash login shell
tags: tutorials
run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell'
status:
regex:
exp: "^Login Shell$"
stream: stdout
bash_nonlogin_shebang:
type: script
executor: generic.local.bash
shebang: "#!/bin/bash"
description: customize shebang line with default bash (nonlogin) shell
tags: tutorials
run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell'
status:
regex:
exp: "^Not Login Shell$"
stream: stdout
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
regex_on_multiple_files:
type: script
executor: generic.local.bash
description: Test regex on multiple files
run: |
echo "Hello" > hello.txt
buildtest --help > buildtest_help.txt
status:
file_regex:
- file: hello.txt
exp: '^(Hello)$'
- file: buildtest_help.txt
exp: '^(usage: buildtest)'
regex_on_directory_not_supported:
type: script
executor: generic.local.bash
description: Test regex on directory is not supported
run: |
mkdir -p hello
echo "Hello" > hello/hello.txt
status:
file_regex:
- file: hello
exp: '^(Hello)$'
file_expansion_supported:
type: script
executor: generic.local.bash
description: Test regex with variable and shell expansion
run: |
echo "Hello" > $BUILDTEST_ROOT/hello.txt
echo "Hello" > $HOME/hello.txt
status:
file_regex:
- file: $BUILDTEST_ROOT/hello.txt
exp: '^(Hello)$'
- file: ~/hello.txt
exp: '^(Hello)$'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
post_run_example:
type: script
executor: generic.local.bash
description: post run example that will remove symbolic link
run: |
ln -s $HOME/.bashrc $HOME/.bashrc_link
mkdir demo
post_run: |
unlink $HOME/.bashrc_link
rmdir demo
status:
is_dir:
- demo
is_symlink:
- $HOME/.bashrc_link
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_range_ex:
type: script
executor: generic.local.bash
description: Example on assert_range
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_range:
comparisons:
- name: copy
lower: 5000
upper: 20000
- name: scale
lower: 4500
upper: 20000
- name: add
lower: 4300
upper: 20000
- name: triad
lower: 5600
upper: 20000
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
always_pass:
type: script
executor: 'generic.local.sh'
description: This test will always 'PASS'
run: exit 1
status:
state: PASS
always_fail:
type: script
executor: 'generic.local.sh'
description: This test will always 'FAIL'
run: exit 0
status:
state: FAIL
test_fail_returncode_match:
type: script
executor: 'generic.local.sh'
description: This test will 'FAIL' even if we have returncode match
run: exit 1
status:
state: FAIL
returncode: 1
test_pass_returncode_mismatch:
type: script
executor: 'generic.local.sh'
description: This test will 'PASS' even if we have returncode mismatch
run: exit 1
status:
state: PASS
returncode: 2
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
string_tag:
type: script
executor: generic.local.bash
description: tags can be a string
tags: network
run: hostname
list_of_strings_tags:
type: script
executor: generic.local.bash
description: tags can be a list of strings
tags: [network, ping]
run: ping -c 4 www.google.com
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
executors_metrics_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring metrics by executors section
tags: [tutorials]
run: echo "Hello World"
executors:
generic.local.bash:
metrics:
hello:
type: str
regex:
stream: stdout
exp: "(Hello)"
generic.local.sh:
metrics:
world:
type: str
regex:
stream: stdout
exp: "(World)"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_gt_example:
type: script
executor: generic.local.bash
description: Run stream test with metrics example using assert greater than.
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_gt:
comparisons:
- name: copy
ref: 5000
- name: scale
ref: 5500
- name: add
ref: 6000
- name: triad
ref: 6500
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
_bin_sh_shell:
executor: generic.local.sh
type: script
description: "/bin/sh shell example"
shell: /bin/sh
tags: [tutorials]
run: "bzip2 --help"
_bin_bash_shell:
executor: generic.local.bash
type: script
description: "/bin/bash shell example"
shell: /bin/bash
tags: [tutorials]
run: "bzip2 -h"
bash_shell:
executor: generic.local.bash
type: script
description: "bash shell example"
shell: bash
tags: [tutorials]
run: "echo $SHELL"
sh_shell:
executor: generic.local.sh
type: script
description: "sh shell example"
shell: sh
tags: [tutorials]
run: "echo $SHELL"
shell_options:
executor: generic.local.sh
type: script
description: "shell options"
shell: "sh -x"
tags: [tutorials]
run: |
echo $SHELL
hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
metric_regex_example_with_re:
executor: generic.local.bash
type: script
description: capture metric with different regex types
tags: tutorials
run: |
echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
echo "HPCG result is INVALID with a GFLOP/s rating of=28.1215" > hpcg.txt
metrics:
hpcg_result:
type: float
regex:
re: "re.search"
exp: '(\d+\.\d+)$'
stream: stdout
hpcg_text:
type: str
regex:
re: "re.match"
exp: '^HPCG result is VALID'
stream: stdout
hpcg_file_text:
type: str
file_regex:
re: "re.match"
exp: '^HPCG result is INVALID'
file: hpcg.txt
hpcg_file_result:
type: float
file_regex:
re: "re.search"
exp: '(\d+\.\d+)$'
file: hpcg.txt
status:
assert_eq:
comparisons:
- name: hpcg_text
ref: "HPCG result is VALID"
- name: hpcg_result
ref: 63.6515
- name: hpcg_file_text
ref: "HPCG result is INVALID"
- name: hpcg_file_result
ref: 28.1215
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
sbatch_example:
type: script
executor: generic.local.bash
description: This test runs hostname using sbatch directives
sbatch:
- "-t 10:00:00"
- "-p normal"
- "-N 1"
- "-n 8"
run: hostname
bsub_example:
type: script
executor: generic.local.bash
description: This test runs hostname using bsub directives
bsub:
- "-W 00:30"
- "-N 1"
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
declare_shell_bin_zsh:
executor: generic.local.zsh
type: script
description: declare shell zsh
shell: "zsh"
run: hostname
declare_shell_zsh:
executor: generic.local.zsh
type: script
description: declare shell /bin/zsh
shell: "/bin/zsh"
run: hostname
declare_shell_bin_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/csh
shell: "/bin/csh"
run: hostname
declare_shell_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "csh"
run: hostname
declare_shell_bin_tcsh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "/bin/tcsh"
run: hostname
declare_shell_tcsh:
executor: generic.local.csh
type: script
description: declare shell tcsh
shell: "tcsh"
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
pass_test:
type: script
executor: generic.local.bash
description: This test will always pass
status:
state: PASS
run: |
echo "This test will pass"
sleep 2
fail_test:
type: script
executor: generic.local.bash
description: This test will run if test 'pass_test' is in state 'PASS'
status:
state: FAIL
needs:
- pass_test:
state: PASS
run: |
echo "This test will fail"
sleep 2
pass_and_fail_test:
type: script
executor: generic.local.bash
description: This test will run if pass_test is 'PASS' and fail_test is 'FAI
needs:
- pass_test:
state: PASS
- fail_test:
state: FAIL
run: |
sleep 2
exit 1
status:
returncode: 1
final_test:
type: script
executor: generic.local.bash
description: Test will run after 'pass_test', 'fail_test', and 'pass_and_fai
needs:
- pass_test
- fail_test
- pass_and_fail_test:
returncode: 1
run: |
sleep 2
echo "Hello world"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
exit1_fail:
executor: generic.local.bash
type: script
description: exit 1 by default is FAIL
tags: [tutorials, fail]
run: exit 1
exit1_pass:
executor: generic.local.bash
type: script
description: report exit 1 as PASS
run: exit 1
tags: [tutorials, pass]
status:
returncode: [1]
returncode_list_mismatch:
executor: generic.local.bash
type: script
description: exit 2 failed since it failed to match returncode 1
run: exit 2
tags: [tutorials, fail]
status:
returncode: [1, 3]
returncode_int_match:
executor: generic.local.bash
type: script
description: exit 128 matches returncode 128
run: exit 128
tags: [tutorials, pass]
status:
returncode: 128
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
skip:
type: script
executor: generic.local.bash
description: This test is skipped
skip: Yes
tags: [tutorials]
run: hostname
unskipped:
type: script
executor: generic.local.bash
description: This test is not skipped
skip: No
tags: [tutorials]
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
timelimit_min_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if its within 1.0-
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
max: 3.0
timelimit_min:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if its exceeds min
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
timelimit_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if it's within max
tags: ["tutorials"]
run: sleep 2
status:
runtime:
max: 5.0
timelimit_min_fail:
type: script
executor: generic.local.sh
description: "This test fails because it runs less than mintime of 10 second
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 10.0
timelimit_max_fail:
type: script
executor: generic.local.sh
description: "This test fails because it exceeds maxtime of 1.0 second"
tags: ["tutorials"]
run: sleep 3
status:
runtime:
max: 1.0
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_eq_example:
type: script
executor: generic.local.bash
description: Test for assert equality
vars:
X: 1
Y: 1.5
first: John
last: Smith
run: |
echo "X: $X"
echo "Y: $Y"
echo "Name: $first $last"
metrics:
x:
type: int
regex:
stream: stdout
exp: 'X:\s+(\S+)\s+.*'
item: 1
y:
type: float
regex:
stream: stdout
exp: 'Y:\s+(\S+)\s+.*'
item: 1
first:
type: str
regex:
stream: stdout
exp: 'Name:\s+(\S+)\s+.*'
item: 1
last:
type: str
regex:
stream: stdout
exp: '(Smith)$'
item: 1
status:
assert_eq:
comparisons:
- name: x
ref: 1
- name: y
ref: 1.5
- name: first
ref: John
- name: last
ref: Smith
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
status_exists:
type: script
executor: generic.local.bash
description: status check based for file and directory
run: |
mkdir -p $HOME/dirA
mkdir -p /tmp/ABC
touch file1
status:
exists:
- $HOME/dirA
- ~/.bashrc
- /tmp/ABC
- file1
status_exists_failure:
type: script
executor: generic.local.bash
description: status check failure for existence
run: touch foo
status:
exists:
- bar
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
multiline_run:
executor: generic.local.bash
type: script
description: multiline run command
run: |
echo "1"
echo "2"
single_command_run:
executor: generic.local.bash
type: script
description: single command as a string for run command
run: "hostname"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_ne_example:
type: script
executor: generic.local.bash
description: Test for assert not equal
vars:
X: 1
Y: 1.5
first: John
last: Smith
run: |
echo "X: $X"
echo "Y: $Y"
echo "Name: $first $last"
metrics:
x:
type: int
regex:
stream: stdout
exp: 'X:\s+(\S+)\s+.*'
item: 1
y:
type: float
regex:
stream: stdout
exp: 'Y:\s+(\S+)\s+.*'
item: 1
first:
type: str
regex:
stream: stdout
exp: 'Name:\s+(\S+)\s+.*'
item: 1
last:
type: str
regex:
stream: stdout
exp: '(Smith)$'
item: 1
status:
assert_ne:
comparisons:
- name: x
ref: 2
- name: y
ref: 2.5
- name: first
ref: Robert
- name: last
ref: Brown
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
circle_area:
executor: generic.local.bash
type: script
shell: python
description: "Calculate circle of area given a radius"
tags: [tutorials, python]
run: |
import math
radius = 2
area = math.pi * radius * radius
print("Circle Radius ", radius)
print("Area of circle ", area)
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
executors_vars_env_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring env and vars by executors section
tags: [tutorials]
run: |
echo "X:" $X
echo "Y:" $Y
echo $SHELL
executors:
generic.local.bash:
vars:
X: 1
Y: 3
env:
SHELL: bash
generic.local.sh:
vars:
X: 2
Y: 4
env:
SHELL: sh
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
re.search.stdout:
executor: generic.local.bash
type: script
description: Test re.search on stdout
run: echo "This is a string"
status:
regex:
stream: stdout
exp: 'is'
re: "re.search"
re.match.stdout:
executor: generic.local.bash
type: script
description: Test re.match on stdout
run: echo "This is a string"
status:
regex:
stream: stdout
exp: 'is'
re: "re.match"
re.fullmatch.stdout:
executor: generic.local.bash
type: script
description: Test re.fullmatch on stdout
run: echo "This is a string"
status:
regex:
stream: stdout
exp: 'This is a string'
re: "re.fullmatch"
re.match_on_file_regex:
executor: generic.local.bash
type: script
description: Test re.match on file regex
run: |
echo "This is a string" > file.txt
echo "Hello World" > hello.txt
status:
file_regex:
- file: file.txt
exp: 'string'
re: "re.search"
- file: hello.txt
exp: 'Hello'
re: "re.match"
- file: hello.txt
exp: 'Hello World'
re: "re.fullmatch"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
runtime_test:
type: script
executor: generic.local.bash
description: This test will sleep 5 second but will fail due to runtime 2sec
status:
runtime:
min: 2.0
run: sleep 5
runtime_test_pass:
type: script
executor: generic.local.bash
description: This test will run when runtime_test_pass is PASS
needs:
- runtime_test:
state: PASS
run: echo "Performing some action when test PASS"
runtime_test_fail:
type: script
executor: generic.local.bash
description: This test will run when runtime_test_pass is FAIL
needs:
- runtime_test:
state: FAIL
run: echo "Performing some action when test FAIL"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
declare_vars:
executor: generic.local.bash
type: script
description: declaring variables
vars:
First: Bob
Last: Bill
run: |
echo "First:" $First
echo "Last:" $Last
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
status_returncode_by_executors:
type: script
executor: "generic.local.(bash|sh)"
description: define status per executor type.
tags: [tutorials]
run: exit 0
executors:
generic.local.bash:
status:
returncode: [0, 2]
generic.local.sh:
status:
returncode: 1
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
bash_env_variables:
executor: generic.local.bash
description: Declare environment variables in default shell (bash)
type: script
env:
FIRST_NAME: avocado
LAST_NAME: dinosaur
tags: [tutorials]
run: |
hostname
whoami
echo $USER
printf "${FIRST_NAME} ${LAST_NAME}\n"
csh_env_declaration:
executor: generic.local.csh
type: script
description: "csh shell example to declare environment variables"
shell: /bin/csh
tags: [tutorials]
env:
SHELL_NAME: "csh"
run: echo "This is running $SHELL_NAME"
tcsh_env_declaration:
executor: generic.local.csh
type: script
description: "tcsh shell example to declare environment variables"
shell: /bin/tcsh
tags: [tutorials]
env:
path: "/usr/local/bin:$PATH"
run: echo $path
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
linecount_stdout:
type: script
executor: generic.local.bash
description: "Write 10 lines to stdout and run linecount check"
run: |
for i in {1..10}; do
echo $i
done
status:
linecount:
stream: stdout
count: 10
linecount_stderr_mismatch:
type: script
executor: generic.local.bash
description: "Write 10 lines to stderr and run linecount check"
run: |
for i in {1..10}; do
echo $i >&2
done
status:
linecount:
stream: stderr
count: 5
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
symlink_test:
type: script
executor: generic.local.bash
description: status check based on symbolic link
run: |
ln -s /tmp scratch
ln -s $HOME/.bashrc $HOME/.bashrc_link
status:
is_symlink:
- scratch
- $HOME/.bashrc_link
- ~/.bashrc_link
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
hello_world_docker:
type: script
executor: generic.local.bash
description: run hello-world container with docker
container:
platform: "docker"
image: hello-world
run: echo 'Test Complete!'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
executors_sbatch_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring env and vars by executors section
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_by_expression:
type: script
executor: generic.local.bash
description: file count by expression
run: |
mkdir -p /tmp
touch /tmp/foo{1..5}.txt
ls -l /tmp/foo*.txt
filenames=$(find $BUILDTEST_ROOT/buildtest -type f \( -name "defaults.py"
totalfiles=$(find $BUILDTEST_ROOT/buildtest -type f \( -name "defaults.py
echo "Filenames: $filenames"
echo "Total files: $totalfiles"
status:
file_count:
- dir: /tmp
filepattern: 'foo[1-5].txt$'
count: 5
- dir: '$BUILDTEST_ROOT/buildtest'
filepattern: '(defaults|main).py$'
count: 2
depth: 1
file_extension_and_filepattern:
type: script
executor: generic.local.bash
description: file count by file extension and file pattern
run: |
touch foo{1..5}.txt
touch {conf,main}.py
ls -l
status:
file_count:
- dir: .
ext: '.txt'
filepattern: '(conf|main).py$'
count: 7
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
jobA:
type: script
executor: generic.local.bash
description: no job dependency
run: |
echo "This job has no dependency"
sleep 5
jobB:
type: script
executor: generic.local.bash
description: job dependency on jobA
needs: [jobA]
run: |
echo "This job depends on jobA"
sleep 2
jobC:
type: script
executor: generic.local.bash
description: job dependency on jobA and jobB
needs: [jobA, jobB]
run: |
echo "This job depends on jobA and jobB"
sleep 2
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_and_dir_checks:
type: script
executor: generic.local.bash
description: status check for files and directories
run: hostname
status:
is_dir:
- $HOME
- $HOME/.bashrc
- /tmp
combined_file_and_dir_checks:
type: script
executor: generic.local.bash
description: status check for files and directories
run: hostname
status:
is_dir:
- $HOME
- /tmp
is_file:
- $HOME/.bashrc
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
bind_mount_in_container:
type: script
executor: generic.local.bash
description: run a python script in container
container:
platform: "docker"
image: ubuntu:latest
mounts: "/tmp:/tmp"
command: bash -c "echo 'hello world' > /tmp/hello.txt"
run: |
cat /tmp/hello.txt
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
stream_test:
type: script
executor: generic.local.bash
description: Run stream test with metrics example using assert greater equal
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_ge:
mode: and
comparisons:
- name: copy
ref: 5000
- name: scale
ref: 5500
- name: add
ref: 6000
- name: triad
ref: 6500
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
assert_le_example:
type: script
executor: generic.local.bash
description: Run stream test with metrics example using assert less than equ
env:
OMP_NUM_THREADS: 4
run: |
wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
gcc -openmp -o stream stream.c
./stream
metrics:
copy:
type: float
regex:
exp: 'Copy:\s+(\S+)\s+.*'
stream: stdout
item: 1
scale:
type: float
regex:
exp: 'Scale:\s+(\S+)\s+.*'
stream: stdout
item: 1
add:
type: float
regex:
exp: 'Add:\s+(\S+)\s+.*'
stream: stdout
item: 1
triad:
type: float
regex:
exp: 'Triad:\s+(\S+)\s+.*'
stream: stdout
item: 1
status:
assert_le:
comparisons:
- name: copy
ref: 5000
- name: scale
ref: 5500
- name: add
ref: 6000
- name: triad
ref: 6500
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
test1:
type: script
executor: generic.local.bash
description: This test will pass with exit 1
run: exit 1
status:
state: PASS
test2:
type: script
executor: generic.local.bash
description: This test will run if test1 has returncode 1
run: exit 2
status:
state: PASS
needs:
- test1:
returncode: 1
test3:
type: script
executor: generic.local.bash
description: This test will run if test1 has returncode 1 and test2 has retu
run: exit 1
status:
state: PASS
needs:
- test1:
returncode: 1
- test2:
returncode: 2
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
contains_and_not_contains:
type: script
executor: generic.local.bash
description: Status check based on contains and not contains where test pass
vars:
X: 1
run: |
echo "X: $X"
metrics:
x:
type: int
regex:
stream: stdout
exp: 'X:\s+(\S+)\s+.*'
item: 1
status:
contains:
comparisons:
- name: x
ref: [1, 2, 4, 8]
not_contains:
comparisons:
- name: x
ref: [2, 4]
assert_contains_fail:
type: script
executor: generic.local.bash
description: Status check based on contains where test fails
vars:
X: 1
run: |
echo "X: $X"
metrics:
x:
type: int
regex:
stream: stdout
exp: 'X:\s+(\S+)\s+.*'
item: 1
status:
contains:
comparisons:
- name: x
ref: ['1', 2, 4, 8]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
summary_example:
type: script
executor: generic.local.bash
description: The summary field can be a multi-line string and exceed 80 char
tags: [tutorials]
summary: |
This is a long description of test that
can exceed 80 characters and be multiline
run: hostname
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_on_directory:
type: script
executor: generic.local.bash
description: file count check in directory
run: |
mkdir -p foo
touch foo/{1..5}
status:
file_count:
- dir: foo
count: 5
file_count_by_extension:
type: script
executor: generic.local.bash
description: file count by extension
run: |
mkdir -p foo/bar
touch foo/{1..5}.sh
touch foo/bar/{1..3}.py foo/bar/{1..3}.txt
status:
file_count:
- dir: foo
ext: '.sh'
depth: 1
count: 5
- dir: foo/bar
ext: ['.py', '.txt']
count: 6
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
file_count_by_filetype:
type: script
executor: generic.local.bash
description: Count the number of directories and symbolic links
run: |
mkdir -p foo/{bar,baz}
find foo -type dir
ln -s foo/bar foo/bar.link
ln -s foo/baz foo/baz.link
status:
file_count:
- dir: foo
count: 3
filetype: 'dir'
- dir: foo
count: 2
filetype: 'symlink'
Spack Schema Examples
buildtest schema -n spack.schema.json --example
$ buildtest schema -n spack.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
specs_must_be_list_of_strings:
type: spack
executor: generic.local.sh
description: 'specs must be a list of strings'
spack:
root: $HOME/spack
env:
create:
name: myproject
specs: [1, zlib]
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_load_invalid_example:
type: spack
executor: generic.local.bash
description: "Run spack load for m4 package and run tests"
tags: [spack]
spack:
root: $HOME/spack
load:
options: '--only package'
specs: [1.5]
test:
run:
specs: ['m4']
results:
option: "-l"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_env_deactivate_first:
type: spack
executor: generic.local.bash
description: "deactivate a spack environment first prior to activating it"
tags: [spack]
spack:
root: $HOME/spack
env:
create:
dir: $HOME/spack-envs/m4
deactivate: 2.0
activate:
dir: $HOME/spack-envs/m4
specs:
- 'm4'
install:
option: ''
post_cmds: |
spack find
rm -rf $HOME/spack-envs/m4
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_test_run_invalid_spec:
type: spack
executor: generic.local.sh
description: "specs property requires a list of strings. "
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib']
test:
remove_tests: true
run:
specs: ['m4', 1]
results:
option: '-f'
post_cmds: |
spack find
rm -rf $SPACK_ROOT
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
invalid_type_mirror_field:
type: spack
executor: generic.local.sh
description: The mirror field must be a key value pair
spack:
root: $HOME/spack
mirror: https://caches.e4s.io
env:
create:
name: myproject
specs:
- zlib
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_test_additionalProperties:
type: spack
executor: generic.local.sh
description: "Check for additionalProperties in test section. FOO key is not
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib']
test:
FOO: BAR
remove_tests: true
run:
specs: ['m4', 'zlib']
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
additionalProperties_spack_field:
type: spack
executor: generic.local.sh
description: additional Properties can't be specified in spack section
spack:
root: $HOME/spack
FOO: BAR
env:
create:
name: myproject
specs:
- zlib
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_create_from_manifest:
type: spack
executor: generic.local.sh
description: Create spack enviromment from manifest file
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_mirror:
type: spack
executor: generic.local.sh
description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in e
spack:
root: $HOME/spack/
env:
mirror:
h5: /path/to/mirror
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
install_without_env:
type: spack
executor: generic.local.sh
description: install specs without environment
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
pre_post_cmd_spack_install:
type: spack
executor: generic.local.sh
description: run commands before and after spack using pre_cmds and post_cmd
pre_cmds: |
cd $HOME
git clone https://github.com/spack/spack
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
post_cmds: |
spack find
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_create_directory:
type: spack
executor: generic.local.sh
description: Create spack environment by directory
spack:
root: $HOME/spack/
env:
create:
dir: $HOME/spack-env/myproject
specs:
- 'm4'
- 'zlib@1.2.11'
install:
options: '--cache-only'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_concretized_install:
type: spack
description: run 'spack concretize -f' in an environment and install specs
executor: generic.local.sh
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
concretize: true
install:
options: '--cache-only'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_activate:
type: spack
executor: generic.local.sh
description: Activate spack environment by name
spack:
root: $HOME/spack/
env:
activate:
name: myproject
specs:
- m4
- zlib
install:
options: ''
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
vars_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "variable declaration with multiple executors"
tags: [spack]
executors:
generic.local.sh:
vars:
FOO: BAR
generic.local.bash:
vars:
HELLO: WORLD
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
sbatch_field_with_spack:
type: spack
executor: generic.local.sh
description: Specify sbatch field with spack schema
sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"]
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
env_create_name:
type: spack
executor: generic.local.sh
description: Create spack environment by name
spack:
root: $HOME/spack/
env:
create:
name: myproject
specs:
- m4
- zlib
install:
options: ''
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_env_deactivate_first:
type: spack
executor: generic.local.bash
description: "deactivate a spack environment first prior to activating it"
tags: [spack]
spack:
root: $HOME/spack
env:
create:
dir: $HOME/spack-envs/m4
deactivate: true
activate:
dir: $HOME/spack-envs/m4
specs:
- 'm4'
install:
option: ''
post_cmds: |
spack find
rm -rf $HOME/spack-envs/m4
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_sbatch_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "sbatch directives can be defined in spack schema"
tags: [spack]
executors:
generic.local.sh:
sbatch: ["-N 1", "-t 20"]
generic.local.bash:
sbatch: ["-N 8", "-t 10"]
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
skip_test_in_spack:
skip: True
type: spack
executor: generic.local.sh
description: This test will be skipped
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
var_declaration_in_spack:
type: spack
executor: generic.local.sh
description: Define variables and environment variables in spack
vars:
FOO: "BAR"
env:
SPACK_GNUPGHOME: "$HOME/.gnupg"
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
remove_spack_environment_automatically:
type: spack
executor: generic.local.sh
description: remove spack environment automatically
spack:
root: $HOME/spack
env:
create:
remove_environment: true
name: m4
activate:
name: m4
specs:
- 'm4'
post_cmds: |
spack find
rm -rf $SPACK_ROOT
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
remove_spack_environment:
type: spack
executor: generic.local.sh
description: remove spack environment explicitly before creating environment
spack:
root: $HOME/spack
env:
rm:
name: m4
create:
name: m4
activate:
name: m4
specs:
- 'm4'
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/latest… ─
buildspecs:
spack_load_valid_example:
type: spack
executor: generic.local.bash
description: "Run spack load for m4 package and run tests"
tags: [spack]
spack:
root: $HOME/spack
load:
options: '--only package'
specs: [m4]
test:
run:
specs: ['m4']
results:
option: "-l"