Buildtest Schemas¶
CLI for buildtest schemas (buildtest schema
)¶
buildtest uses JSON Schema for validating buildspecs and buildtest configuration file.
You can use buildtest schema
command to see the list of schemas
supported by buildtest. The schema files are denoted by .schema.json
file extension.
$ buildtest schema
global.schema.json
definitions.schema.json
settings.schema.json
compiler-v1.0.schema.json
spack-v1.0.schema.json
script-v1.0.schema.json
Shown below is the command usage of buildtest schema
$ buildtest schema --help
usage: buildtest [options] [COMMANDS] schema [-h] [-e] [-j] [-n Schema Name]
optional arguments:
-h, --help show this help message and exit
-e, --example Show schema examples
-j, --json Display json schema file
-n Schema Name, --name Schema Name
show schema by name (e.g., script)
The json schemas are published at https://buildtesters.github.io/buildtest/ and we
provide a command line interface to view schema files and examples. You must use the
--name
option to select a schema, for instance if you want to view the JSON Schema for
script-v1.0.schema.json you can run the following:
buildtest schema --name script-v1.0.schema.json --json
Schema Naming Convention¶
All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.
Schema Files¶
Definition Schema¶
This schema is used for declaring definitions that need to be
reused in multiple schemas. We use $ref
keyword to reference definitions from this file.
{
"$id": "definitions.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "JSON Schema Definitions File. ",
"description": "This file is used for declaring definitions that are referenced from other schemas",
"definitions": {
"list_of_strings": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"string_or_list": {
"oneOf": [
{
"type": "string"
},
{
"$ref": "#/definitions/list_of_strings"
}
]
},
"list_of_ints": {
"type": "array",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "integer"
}
},
"int_or_list": {
"oneOf": [
{
"type": "integer"
},
{
"$ref": "#/definitions/list_of_ints"
}
]
},
"regex": {
"type": "object",
"description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
"required": [
"stream",
"exp"
],
"additionalProperties": false,
"properties": {
"stream": {
"type": "string",
"enum": [
"stdout",
"stderr"
],
"description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
},
"exp": {
"type": "string",
"description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
}
}
},
"env": {
"type": "object",
"description": "One or more key value pairs for an environment (key=value)",
"minItems": 1,
"items": {
"type": "object",
"minItems": 1,
"propertyNames": {
"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
}
}
},
"description": {
"type": "string",
"description": "The ``description`` field is used to document what the test is doing",
"maxLength": 80
},
"tags": {
"description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
"$ref": "#/definitions/string_or_list"
},
"skip": {
"type": "boolean",
"description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
},
"executor": {
"type": "string",
"description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
},
"metrics_field": {
"type": "object",
"additionalProperties": false,
"properties": {
"regex": {
"$ref": "#/definitions/regex"
},
"vars": {
"type": "string",
"description": "Assign value to metric based on variable name"
},
"env": {
"type": "string",
"description": "Assign value to metric based on environment variable"
}
}
},
"metrics": {
"type": "object",
"description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/metrics_field",
"description": "Name of metric"
}
}
},
"run_only": {
"type": "object",
"description": "A set of conditions to specify when running tests. All conditions must pass in order to process test.",
"additionalProperties": false,
"properties": {
"scheduler": {
"type": "string",
"description": "Test will run only if scheduler is available. We assume binaries are available in $PATH",
"enum": [
"lsf",
"slurm",
"cobalt",
"pbs"
]
},
"user": {
"type": "string",
"description": "Test will run only if current user matches this field, otherwise test will be skipped"
},
"platform": {
"type": "string",
"description": "This test will run if target system is Linux or Darwin. We check target system using ``platform.system()`` and match with input field",
"enum": [
"Linux",
"Darwin"
]
},
"linux_distro": {
"type": "array",
"description": "Specify a list of Linux Distros to check when processing test. If target system matches one of input field, test will be processed.",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string",
"enum": [
"darwin",
"ubuntu",
"debian",
"rhel",
"centos",
"fedora",
"sles",
"opensuse",
"amazon",
"arch"
]
}
}
}
},
"batch": {
"type": "object",
"description": "The ``batch`` field is used to specify scheduler agnostic directives that are translated to #SBATCH or #BSUB based on your scheduler. This is an experimental feature that supports a subset of scheduler parameters.",
"additionalProperties": false,
"properties": {
"account": {
"type": "string",
"description": "Specify Account to charge job"
},
"begintime": {
"type": "string",
"description": "Specify begin time when job will start allocation"
},
"cpucount": {
"type": "string",
"description": "Specify number of CPU to allocate"
},
"email-address": {
"type": "string",
"description": "Email Address to notify on Job State Changes"
},
"exclusive": {
"type": "boolean",
"description": "Specify if job needs to run in exclusive mode"
},
"memory": {
"type": "string",
"description": "Specify Memory Size for Job"
},
"network": {
"type": "string",
"description": "Specify network resource requirement for job"
},
"nodecount": {
"type": "string",
"description": "Specify number of Nodes to allocate"
},
"qos": {
"type": "string",
"description": "Specify Quality of Service (QOS)"
},
"queue": {
"type": "string",
"description": "Specify Job Queue"
},
"tasks-per-core": {
"type": "string",
"description": "Request number of tasks to be invoked on each core. "
},
"tasks-per-node": {
"type": "string",
"description": "Request number of tasks to be invoked on each node. "
},
"tasks-per-socket": {
"type": "string",
"description": "Request the maximum tasks to be invoked on each socket. "
},
"timelimit": {
"type": "string",
"description": "Specify Job timelimit"
}
}
},
"status": {
"type": "object",
"description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
"additionalProperties": false,
"properties": {
"slurm_job_state": {
"type": "string",
"enum": [
"COMPLETED",
"FAILED",
"OUT_OF_MEMORY",
"TIMEOUT"
],
"description": "This field can be used for checking Slurm Job State, if there is a match buildtest will report as ``PASS`` "
},
"returncode": {
"description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
"$ref": "#/definitions/int_or_list"
},
"regex": {
"$ref": "#/definitions/regex",
"description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
},
"runtime": {
"type": "object",
"description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
"properties": {
"min": {
"type": "number",
"minimum": 0,
"description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
},
"max": {
"type": "number",
"minimum": 0,
"description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
}
}
}
}
},
"BB": {
"$ref": "#/definitions/list_of_strings",
"description": "Create burst buffer space, this specifies #BB options in your test."
},
"DW": {
"$ref": "#/definitions/list_of_strings",
"description": "Specify Data Warp option (#DW) when using burst buffer."
},
"sbatch": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #SBATCH options in test script."
},
"bsub": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #BSUB options in test script."
},
"cobalt": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #COBALT options in test script."
},
"pbs": {
"$ref": "#/definitions/list_of_strings",
"description": "This field is used for specifying #PBS directives in test script."
},
"executors": {
"type": "object",
"description": "Define executor specific configuration",
"patternProperties": {
"description": "Name of executor to override configuration",
"^.*$": {
"additionalProperties": false,
"properties": {
"env": { "$ref": "#/definitions/env" },
"vars": { "$ref": "#/definitions/env" },
"sbatch": { "$ref": "#/definitions/list_of_strings" },
"bsub": { "$ref": "#/definitions/list_of_strings" },
"pbs": { "$ref": "#/definitions/list_of_strings" },
"cobalt": { "$ref": "#/definitions/list_of_strings" },
"BB": { "$ref": "#/definitions/BB" },
"DW": { "$ref": "#/definitions/DW" },
"status": { "$ref": "#/definitions/status" },
"metrics": { "$ref": "#/definitions/metrics" }
}
}
}
}
}
}
Settings Schema¶
This schema defines how buildtest configuration file is validated.
{
"$id": "settings.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "buildtest configuration schema",
"type": "object",
"required": [
"system"
],
"additionalProperties": false,
"properties": {
"system": {
"type": "object",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/system"
}
}
}
},
"definitions": {
"system": {
"required": [
"executors",
"moduletool",
"load_default_buildspecs",
"hostnames",
"compilers"
],
"additionalProperties": false,
"type": "object",
"properties": {
"hostnames": {
"type": "array",
"description": "Specify a list of hostnames to check where buildtest can run for the given system record",
"items": {
"type": "string"
}
},
"description": {
"type": "string",
"description": "system description field"
},
"buildspec_roots": {
"type": "array",
"items": {
"type": "string"
},
"description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
},
"load_default_buildspecs": {
"type": "boolean",
"description": "Specify whether buildtest should automatically load buildspecs provided in buildtest repo into buildspec cache"
},
"testdir": {
"type": "string",
"description": "Specify full path to test directory where buildtest will write tests."
},
"logdir": {
"type": "string",
"description": "Specify location where buildtest will write log files"
},
"moduletool": {
"type": "string",
"description": "Specify modules tool used for interacting with ``module`` command. ",
"enum": [
"environment-modules",
"lmod",
"N/A"
]
},
"processor": {
"type": "object",
"description": "Specify processor information",
"additionalProperties": false,
"properties": {
"numcpus": {"type": "integer", "minimum": 1, "description": "Specify Total Number of CPUs"},
"sockets": {"type": "integer", "minimum": 1, "description": "Specify Number of CPU Sockets"},
"cores": {"type": "integer", "minimum": 1, "description": "Specify Number of Physical Cores"},
"threads_per_core": {"type": "integer", "minimum": 1, "description": "Specify Threads per Core" },
"core_per_socket": {"type": "integer", "minimum": 1, "description": "Specify Cores per Socket"},
"model": {"type": "string", "description": "Specify Processor Model" },
"arch": {"type": "string", "description": "Specify processor architecture"},
"vendor": {"type": "string", "description": "Vendor Name"}
}
},
"compilers": {
"type": "object",
"description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
"additionalProperties": false,
"properties": {
"find": {
"type": "object",
"additionalProperties": false,
"description": "Find compilers by specifying regular expression that is applied to modulefile names",
"properties": {
"gcc": {
"type": "string",
"description": "Specify a regular expression to search for gcc compilers from your module stack"
},
"intel": {
"type": "string",
"description": "Specify a regular expression to search for intel compilers from your module stack"
},
"cray": {
"type": "string",
"description": "Specify a regular expression to search for cray compilers from your module stack"
},
"clang": {
"type": "string",
"description": "Specify a regular expression to search for clang compilers from your module stack"
},
"cuda": {
"type": "string",
"description": "Specify a regular expression to search for cuda compilers from your module stack"
},
"pgi": {
"type": "string",
"description": "Specify a regular expression to search for pgi compilers from your module stack"
},
"upcxx": {
"type": "string",
"description": "Specify a regular expression to search for upcxx compilers from your module stack"
}
}
},
"compiler": {
"type": "object",
"additionalProperties": false,
"description": "Start of compiler declaration",
"properties": {
"gcc": {
"description": "Declaration of one or more GNU compilers where we define C, C++ and Fortran compiler. The GNU compiler wrapper are ``gcc``, ``g++`` and ``gfortran``. ",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"intel": {
"description": "Declaration of one or more Intel compilers where we define C, C++ and Fortran compiler. The Intel compiler wrapper are ``icc``, ``icpc`` and ``ifort``. ",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cray": {
"description": "Declaration of one or more Cray compilers where we define C, C++ and Fortran compiler. The Cray compiler wrapper are ``cc``, ``CC`` and ``ftn``.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"pgi": {
"description": "Declaration of one or more PGI compilers where we define C, C++ and Fortran compiler. The PGI compiler wrapper are ``pgcc``, ``pgc++`` and ``pgfortran``.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"clang": {
"description": "Declaration of one or more Clang compilers where we define C, C++ compiler. The Clang compiler wrapper are ``clang``, ``clang++``.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"cuda": {
"description": "Declaration of one or more Cuda compilers where we define C compiler. The Cuda compiler wrapper is ``nvcc``. ",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
},
"upcxx": {
"description": "Declaration of one or more UPCXX compilers where we define C, C++ compiler. The UPCXX compiler wrapper are ``upcxx``.",
"type": "object",
"patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
}
}
}
}
},
"executors": {
"type": "object",
"additionalProperties": false,
"description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``, ``cobalt``. The executors are referenced in buildspec using ``executor`` field.",
"properties": {
"defaults": {
"type": "object",
"description": "Specify default executor settings for all executors",
"additionalProperties": false,
"properties": {
"pollinterval": {
"type": "integer",
"description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
"minimum": 10,
"maximum": 300,
"default": 30
},
"launcher": {
"type": "string",
"enum": [
"sbatch",
"bsub",
"qsub"
],
"description": "Specify batch launcher to use when submitting jobs, this is applicable for LSF and Slurm Executors."
},
"max_pend_time": {
"$ref": "#/definitions/max_pend_time"
},
"account": {
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"}
}
},
"local": {
"type": "object",
"description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/local"
}
}
},
"lsf": {
"type": "object",
"description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/lsf"
}
}
},
"slurm": {
"type": "object",
"description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/slurm"
}
}
},
"cobalt": {
"type": "object",
"description": "The ``cobalt`` section is used for declaring Cobalt executors for running jobs using Cobalt scheduler",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/cobalt"
}
}
},
"pbs": {
"type": "object",
"description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
"patternProperties": {
"^.*$": {
"$ref": "#/definitions/pbs"
}
}
}
}
},
"cdash": {
"type": "object",
"description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
"required": ["url", "project", "site"],
"properties": {
"url": {
"type": "string",
"description": "Url to CDASH server"
},
"project": {
"type": "string",
"description": "Name of CDASH project"
},
"site": {
"type": "string",
"description": "Site Name reported in CDASH"
}
}
}
}
},
"cc": {
"description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
"type": "string"
},
"cxx": {
"type": "string",
"description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"fc": {
"type": "string",
"description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
},
"max_jobs": {
"description": "Maximum number of jobs that can be run at a given time for a particular executor",
"type": "integer",
"minimum": 1
},
"compiler_section": {
"description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
"type": "object",
"additionalProperties": false,
"required": [ "cc", "cxx", "fc" ],
"properties": {
"cc": { "$ref": "#/definitions/cc" },
"cxx": { "$ref": "#/definitions/cxx" },
"fc": { "$ref": "#/definitions/fc" },
"module": { "$ref": "#/definitions/module" }
}
},
"unique_string_array": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"disable": {"type": "boolean", "description": "Disable executor"},
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": { "type": "string" }
}
}
},
"script": {
"type": "array",
"additionalProperties": false,
"items": { "type": "string" }
},
"max_pend_time": {
"type": "integer",
"description": "Cancel job if it is still pending in queue beyond max_pend_time",
"minimum": 10,
"default": 90
},
"account": {
"type": "string",
"description": "Specify Job Account for charging resources"
},
"local": {
"type": "object",
"description": "An instance object of local executor",
"additionalProperties": false,
"required": [ "shell" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"shell": {
"type": "string",
"description": "Specify the shell launcher you want to use when running tests locally",
"pattern": "^(/bin/bash|/bin/sh|/bin/csh|/bin/tcsh|/bin/zsh|sh|bash|csh|tcsh|zsh|python).*"
},
"before_script": { "#ref": "#/definitions/script" },
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"}
}
},
"slurm": {
"type": "object",
"additionalProperties": false,
"description": "An instance object of slurm executor",
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "sbatch" ],
"description": "Specify the slurm batch scheduler to use. This overrides the default ``launcher`` field. This must be ``sbatch``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
},
"cluster": {
"type": "string",
"description": "Specify the slurm cluster you want to use ``-M <cluster>``"
},
"partition": {
"type": "string",
"description": "Specify the slurm partition you want to use ``-p <partition>``"
},
"qos": {
"type": "string",
"description": "Specify the slurm qos you want to use ``-q <qos>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"max_pend_time": {
"description": "overrides default ``max_pend_time`` value",
"$ref": "#/definitions/max_pend_time"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"}
}
},
"lsf": {
"type": "object",
"description": "An instance object of lsf executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "bsub" ],
"description": "Specify the lsf batch scheduler to use. This overrides the default ``launcher`` field. It must be ``bsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"max_pend_time": {
"description": "overrides default ``max_pend_time`` value",
"$ref": "#/definitions/max_pend_time"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"}
}
},
"cobalt": {
"type": "object",
"description": "An instance object of cobalt executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "qsub" ],
"description": "Specify the cobalt batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"max_pend_time": {
"description": "overrides default ``max_pend_time`` value",
"$ref": "#/definitions/max_pend_time"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"}
}
},
"pbs": {
"type": "object",
"description": "An instance object of cobalt executor",
"additionalProperties": false,
"required": [ "queue" ],
"properties": {
"description": {
"type": "string",
"description": "description field for documenting your executor"
},
"launcher": {
"type": "string",
"enum": [ "qsub" ],
"description": "Specify the pbs batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
},
"options": {
"type": "array",
"items": { "type": "string" },
"description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
},
"queue": {
"type": "string",
"description": "Specify the lsf queue you want to use ``-q <queue>``"
},
"before_script": {
"description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
"#ref": "#/definitions/script"
},
"max_pend_time": {
"description": "overrides default ``max_pend_time`` value",
"$ref": "#/definitions/max_pend_time"
},
"account": {
"description": "overrides default ``account`` value",
"$ref": "#/definitions/account"
},
"max_jobs": {"$ref": "#/definitions/max_jobs"},
"disable": {"$ref": "#/definitions/disable"}
}
}
}
}
Global Schema¶
This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema
{
"$id": "global.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "global schema",
"description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
"type": "object",
"required": ["version","buildspecs"],
"additionalProperties": false,
"properties": {
"version": {
"type": "string",
"description": "The semver version of the schema to use (x.x)."
},
"maintainers": {
"type": "array",
"description": "One or more maintainers or aliases",
"uniqueItems": true,
"minItems": 1,
"items": {
"type": "string"
}
},
"buildspecs": {
"type": "object",
"description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
"propertyNames": {
"pattern": "^[A-Za-z_.][A-Za-z0-9_.]*$",
"maxLength": 32
}
}
}
}
Script Schema¶
This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when
type: script
is specified. For more details on script schema see Script Schema.
{
"$id": "script-v1.0.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "script schema version 1.0",
"description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
"type": "object",
"required": ["type", "run", "executor"],
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^script$",
"description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
},
"description": { "$ref": "definitions.schema.json#/definitions/description" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"BB": { "$ref": "definitions.schema.json#/definitions/BB" },
"DW": { "$ref": "definitions.schema.json#/definitions/DW" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"executor": { "$ref": "definitions.schema.json#/definitions/executor" },
"run_only": { "$ref": "definitions.schema.json#/definitions/run_only" },
"shell": {
"type": "string",
"description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed",
"pattern": "^(/bin/bash|/bin/sh|/bin/csh|/bin/tcsh|/bin/zsh|bash|sh|csh|tcsh|zsh|python).*"
},
"shebang": {
"type": "string",
"description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
},
"run": {
"type": "string",
"description": "A script to run using the default shell."
},
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"skip": { "$ref": "definitions.schema.json#/definitions/skip" },
"tags": { "$ref": "definitions.schema.json#/definitions/tags" },
"metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
"executors": { "$ref": "definitions.schema.json#/definitions/executors" }
}
}
Compiler Schema¶
This is the compiler schema used for validating buildspecs that define test using type: compiler
.
This schema is used for compiling a single source code. For more details see Compiler Schema
{
"$id": "compiler-v1.0.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "compiler schema version 1.0",
"description": "The compiler schema is of ``type: compiler`` in sub-schema which is used for compiling and running programs",
"type": "object",
"required": [
"type",
"source",
"compilers",
"executor"
],
"definitions": {
"cc": {
"type": "string",
"description": "Set C compiler wrapper"
},
"fc": {
"type": "string",
"description": "Set Fortran compiler wrapper"
},
"cxx": {
"type": "string",
"description": "Set C++ compiler wrapper"
},
"cflags": {
"type": "string",
"description": "Set C compiler flags."
},
"fflags": {
"type": "string",
"description": "Set Fortran compiler flags."
},
"cxxflags": {
"type": "string",
"description": "Set C++ compiler flags."
},
"ldflags": {
"type": "string",
"description": "Set linker flags"
},
"cppflags": {
"type": "string",
"description": "Set C or C++ preprocessor flags"
},
"pre_build": {
"type": "string",
"description": "Run commands before building program"
},
"post_build": {
"type": "string",
"description": "Run commands after building program"
},
"pre_run": {
"type": "string",
"description": "Run commands before running program"
},
"post_run": {
"type": "string",
"description": "Run commands after running program"
},
"run": {
"type": "string",
"description": "Run command for launching compiled binary"
},
"default_compiler_all": {
"type": "object",
"description": "Specify compiler configuration for all compiler groups. Use the ``all`` property if configuration is shared across compiler groups. This property can be overridden in compiler group or named compiler in ``config`` section.",
"additionalProperties": false,
"properties": {
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"BB": { "$ref": "definitions.schema.json#/definitions/BB" },
"DW": { "$ref": "definitions.schema.json#/definitions/DW" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" }
}
},
"default_compiler_config": {
"type": "object",
"description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
"properties": {
"cc": { "$ref": "#definitions/cc" },
"fc": { "$ref": "#definitions/fc" },
"cxx": { "$ref": "#definitions/cxx" },
"cflags": { "$ref": "#definitions/cflags" },
"fflags": { "$ref": "#definitions/fflags" },
"cxxflags": { "$ref": "#definitions/cxxflags" },
"ldflags": { "$ref": "#definitions/ldflags" },
"cppflags": { "$ref": "#definitions/cppflags" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs"},
"BB": { "$ref": "definitions.schema.json#/definitions/BB"},
"DW": { "$ref": "definitions.schema.json#/definitions/DW"},
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" }
}
},
"compiler_declaration": {
"type": "object",
"description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
"additionalProperties": false,
"properties": {
"cc": { "$ref": "#definitions/cc" },
"fc": { "$ref": "#definitions/fc" },
"cxx": { "$ref": "#definitions/cxx" },
"cflags": { "$ref": "#definitions/cflags" },
"fflags": { "$ref": "#definitions/fflags" },
"cxxflags": { "$ref": "#definitions/cxxflags" },
"ldflags": { "$ref": "#definitions/ldflags" },
"cppflags": { "$ref": "#definitions/cppflags" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch"},
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub"},
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt"},
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs"},
"BB": { "$ref": "definitions.schema.json#/definitions/BB"},
"DW": { "$ref": "definitions.schema.json#/definitions/DW"},
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"pre_build": { "$ref": "#definitions/pre_build" },
"post_build": { "$ref": "#definitions/post_build" },
"pre_run": { "$ref": "#definitions/pre_run" },
"post_run": { "$ref": "#definitions/post_run" },
"run": { "$ref": "#definitions/run" },
"module": {
"type": "object",
"additionalProperties": false,
"properties": {
"purge": {
"type": "boolean",
"description": "Run ``module purge`` if purge is set"
},
"load": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Load one or more modules via ``module load``"
},
"restore": {
"description": "Load a collection name via ``module restore``",
"type": "string"
},
"swap": {
"description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
"type": "array",
"uniqueItems": true,
"minItems": 2,
"maxItems": 2,
"items": {
"type": "string"
}
}
}
}
}
}
},
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^compiler$",
"description": "Select schema type to use when validating buildspec. This must be of set to ``compiler``."
},
"description": {
"$ref": "definitions.schema.json#/definitions/description"
},
"compilers": {
"type": "object",
"required": [
"name"
],
"additionalProperties": false,
"properties": {
"name": {
"description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"exclude": {
"description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
"$ref": "definitions.schema.json#/definitions/list_of_strings"
},
"default": {
"type": "object",
"additionalProperties": false,
"properties": {
"all": {
"$ref": "#definitions/default_compiler_all"
},
"gcc": {
"$ref": "#definitions/default_compiler_config"
},
"intel": {
"$ref": "#definitions/default_compiler_config"
},
"pgi": {
"$ref": "#definitions/default_compiler_config"
},
"cray": {
"$ref": "#definitions/default_compiler_config"
},
"clang": {
"$ref": "#definitions/default_compiler_config"
},
"cuda": {
"$ref": "#definitions/default_compiler_config"
},
"upcxx": {
"$ref": "#definitions/default_compiler_config"
}
}
},
"config": {
"type": "object",
"description": "Specify compiler configuration based on named compilers.",
"patternProperties": {
"^.*$": {
"$ref": "#definitions/compiler_declaration"
}
}
}
}
},
"source": {
"type": "string",
"description": "Specify a source file for compilation, the file can be relative path to buildspec or an absolute path"
},
"executor": {
"$ref": "definitions.schema.json#/definitions/executor"
},
"run_only": {
"$ref": "definitions.schema.json#/definitions/run_only"
},
"skip": {
"$ref": "definitions.schema.json#/definitions/skip"
},
"tags": {
"$ref": "definitions.schema.json#/definitions/tags"
},
"metrics": {
"$ref": "definitions.schema.json#/definitions/metrics"
}
}
}
Spack Schema¶
This schema is used for writing tests with spack package manager using type: spack
field. For more details
see Spack Schema.
{
"$id": "spack-v1.0.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "spack schema version 1.0",
"description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
"type": "object",
"required": [
"type",
"executor",
"spack"
],
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"pattern": "^spack$",
"description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
},
"description": { "$ref": "definitions.schema.json#/definitions/description" },
"executor": { "$ref": "definitions.schema.json#/definitions/executor" },
"env": { "$ref": "definitions.schema.json#/definitions/env" },
"vars": { "$ref": "definitions.schema.json#/definitions/env" },
"sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
"bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
"cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
"pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
"BB": { "$ref": "definitions.schema.json#/definitions/BB" },
"DW": { "$ref": "definitions.schema.json#/definitions/DW" },
"skip": { "$ref": "definitions.schema.json#/definitions/skip" },
"tags": { "$ref": "definitions.schema.json#/definitions/tags" },
"status": { "$ref": "definitions.schema.json#/definitions/status" },
"metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
"executors": { "$ref": "definitions.schema.json#/definitions/executors" },
"pre_cmds": {
"type": "string",
"description": "Shell commands run before spack"
},
"post_cmds": {
"type": "string",
"description": "Shell commands run after spack"
},
"spack": {
"type": "object",
"description": "Entry point to spack configuration",
"required": [
"root"
],
"additionalProperties": false,
"properties": {
"root": {
"type": "string"
},
"compiler_find": {
"type": "boolean",
"description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror by running ``spack mirror add``"
},
"env": {
"$ref": "#definitions/env",
"description": "Manage spack environments via ``spack env`` command"
},
"install": {
"$ref": "#definitions/install",
"description": "Install spack packages by running ``spack install``. "
},
"verify_spack": {
"type": "boolean",
"description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
"default": true
},
"test": {
"$ref": "#definitions/test",
"description": "Entry point to ``spack test``"
}
}
}
},
"definitions": {
"env": {
"additionalProperties": false,
"type": "object",
"description": "Used for managing spack environment using ``spack env`` command. ",
"properties": {
"create": {
"additionalProperties": false,
"description": "Create a spack environment via ``spack env create``",
"type": "object",
"properties": {
"remove_environment": {
"type": "boolean",
"description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
"default": false
},
"name": {
"type": "string",
"description": "Name of spack environment to create"
},
"manifest": {
"type": "string",
"description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
},
"options": {
"type": "string",
"description": "Pass options to ``spack env create`` command"
},
"dir": {
"type": "string",
"description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
}
}
},
"activate": {
"additionalProperties": false,
"type": "object",
"description": "Activate a spack environment via ``spack env activate``",
"properties": {
"name": {
"type": "string",
"description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
},
"options": {
"type": "string",
"description": "Pass options to ``spack env activate`` command"
},
"dir": {
"type": "string",
"description": "Activate spack environment from directory."
}
}
},
"rm": {
"additionalProperties": false,
"description": "Remove an existing spack environment via ``spack env rm``.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
}
}
},
"mirror": {
"$ref": "definitions.schema.json#/definitions/env",
"description": "Add mirror in spack environment by running ``spack mirror add``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
},
"concretize": {
"type": "boolean",
"description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
}
}
},
"install": {
"description": "Install spack packages using ``spack install`` command",
"additionalProperties": false,
"type": "object",
"properties": {
"options": {
"type": "string",
"description": "Pass options to ``spack install`` command"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to install using ``spack install`` command"
}
}
},
"test": {
"type": "object",
"additionalProperties": false,
"required": [
"run"
],
"properties": {
"remove_tests": {
"type": "boolean",
"description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
},
"run": {
"description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
"type": "object",
"required": [
"specs"
],
"additionalProperties": false,
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test run``"
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "List of specs to run tests by running ``spack test run <specs>``."
}
}
},
"results": {
"type": "object",
"description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
"additionalProperties": false,
"anyOf": [
{"required": ["specs"] },
{"required": ["suite"] },
{"required": ["specs","suite"] }
],
"properties": {
"option": {
"type": "string",
"description": "Pass options to ``spack test results``"
},
"suite": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Report results by suite name by running ``spack test results <suite>``."
},
"specs": {
"$ref": "definitions.schema.json#/definitions/list_of_strings",
"description": "Report result by spec name by running ``spack test run -- <specs>``."
}
}
}
}
}
}
}
Schema Examples¶
The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.
In order to view an example you can run:
buildtest schema -n <schema> --example
Settings Schema Examples¶
$ buildtest schema -n settings.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/slurm-example.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
moduletool: lmod
load_default_buildspecs: True
buildspec_roots:
- $HOME/buildtest-cori
testdir: /tmp/buildtest
executors:
defaults:
pollinterval: 20
launcher: sbatch
max_pend_time: 30
account: admin
slurm:
normal:
options: ["-C haswell"]
qos: normal
before_script: |
time
echo "commands run before job"
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/cobalt-example.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
moduletool: lmod
load_default_buildspecs: True
executors:
defaults:
launcher: qsub
max_pend_time: 30
cobalt:
knl:
queue: knl
haswell:
queue: haswell
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/pbs-example.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
moduletool: N/A
load_default_buildspecs: True
executors:
defaults:
pollinterval: 10
launcher: qsub
max_pend_time: 30
pbs:
workq:
queue: workq
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/lsf-example.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
moduletool: lmod
load_default_buildspecs: False
executors:
defaults:
pollinterval: 10
launcher: bsub
max_pend_time: 45
lsf:
batch:
description: "LSF Executor name 'batch' that submits jobs to 'batch' queue"
queue: batch
account: developer
options: ["-W 20"]
before_script: |
time
echo "commands run before job"
test:
description: "LSF Executor name 'test' that submits jobs to 'test' queue"
launcher: bsub
queue: test
account: qa
options: ["-W 20"]
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
logdir: $BUILDTEST_ROOT/logs
testdir: $BUILDTEST_ROOT/tests
moduletool: N/A
load_default_buildspecs: False
cdash:
url: https://my.cdash.org
project: buildtest
site: laptop
processor:
numcpus: 8
cores: 4
threads_per_core: 2
sockets: 1
model: "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz"
executors:
local:
bash:
description: submit jobs on local machine using bash shell
shell: bash
before_script: |
time
echo "commands run before job"
sh:
description: submit jobs on local machine using sh shell
shell: sh
csh:
description: submit jobs on local machine using csh shell
shell: csh -x
tcsh:
description: submit jobs on local machine using tcsh shell
shell: /bin/tcsh
zsh:
description: submit jobs on local machine using zsh shell
shell: /bin/zsh
python:
description: submit jobs on local machine using python shell
shell: python
compilers:
find:
gcc: "^(gcc|GCC|PrgEnv-gnu)"
intel: "^(intel|Intel|PrgEnv-intel)"
cray: "^(cray|PrgEnv-cray)"
clang: "^(clang|Clang)"
cuda: "^(cuda|CUDA)"
pgi: "^(pgi|PGI|PrgEnv-pgi)"
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
gcc@7.2.0:
cc: 'cc'
cxx: 'cxx'
fc: 'fc'
module:
load:
- gcc/7.2.0
intel:
intel@2019:
cc: 'icc'
cxx: 'icpc'
fc: 'ifort'
module:
purge: True
load:
- gcc/7.2.0
- intel/2019
cray:
craype@2.6.2:
cc: 'cc'
cxx: 'CC'
fc: 'fc'
module:
load: [craype/2.6.2]
swap: [PrgEnv-gnu, PrgEnv-cray]
clang:
clang@12.0.0:
cc: 'clang'
cxx: 'clang++'
fc: 'None'
module:
load: [clang/12.0]
cuda:
cuda@11.0:
cc: 'nvcc'
cxx: 'nvcc'
fc: 'None'
module:
load: [cuda/11.0]
pgi:
pgi@18.0:
cc: 'pgcc'
cxx: 'pgc++'
fc: 'pgfortran'
module:
load: [pgi/18.0]
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/settings.schema.json/valid/combined_executor.yml
________________________________________________________________________________
system:
generic:
hostnames: ['.*']
moduletool: N/A
load_default_buildspecs: True
executors:
local:
bash:
description: submit jobs on local machine
shell: bash -v
slurm:
haswell:
launcher: sbatch
options:
- "-p haswell"
- "-t 00:10"
lsf:
batch:
launcher: bsub
queue: batch
options:
- "-q batch"
- "-t 00:10"
cobalt:
normal:
launcher: qsub
queue: normal
options:
- "-n 1"
- "-t 10"
compilers:
compiler:
gcc:
default:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
fc: /usr/bin/gfortran
Global Schema Examples¶
$ buildtest schema -n global.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
# testing all caps
ABCDEFGHIJKLMNOPQRSTUVWXYZ:
type: script
run: "hostname"
# testing all lowercase letters
abcdefghijklmnopqrstuvwxyz:
type: script
run: "hostname"
# testing '_' in beginning followed by all numbers
_0123456789:
type: script
run: "hostname"
# testing '_' in combination with caps, lowercase and numbers
_ABCDEFabcdef0123456789:
type: script
run: "hostname"
# testing '_' at end of word
abcdefghijklmnopqrstuvwxyz_:
type: script
run: "hostname"
# testing '.' in beginning of word
.helloworld:
type: script
run: hostname
# testing '.' in middle of word
hello.world:
type: script
run: hostname
# testing '.' at end of word
helloworld.:
type: script
run: hostname
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/invalid/missing-version.yml
________________________________________________________________________________
buildspecs:
# Shell would be accepted to indicate a single line shell command (or similar)
login_node_check:
type: script
run: "ping login 1"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/invalid/exceed_testname_length.yml
________________________________________________________________________________
# this test fails because it exceeds 32 character length for test name
version: "1.0"
buildspecs:
this_test_exceeds_32_character_length:
type: script
run: hostname
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/invalid/unique_maintainers.yml
________________________________________________________________________________
version: "1.0"
maintainers: [shahzebsiddiqui, shahzebsiddiqui]
buildspecs:
hostname:
type: script
run: "hostname"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/invalid/maintainers_type_mismatch.yml
________________________________________________________________________________
version: "1.0"
# wrong type for maintainers key, expects a string
maintainers: 1
buildspecs:
hostname:
type: script
run: "hostname"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/global.schema.json/invalid/invalid_pattern.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
# invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$" when declaring a dict
(badname:
type: script
run: "ping login 1"
Script Schema Examples¶
$ buildtest schema -n script-v1.0.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/script-v1.0.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
multiline_run:
executor: generic.local.bash
type: script
description: multiline run command
run: |
echo "1"
echo "2"
single_command_run:
executor: generic.local.bash
type: script
description: single command as a string for run command
run: "hostname"
declare_env:
executor: generic.local.bash
type: script
description: declaring environment variables
env:
FOO: BAR
X: 1
run: |
echo $FOO
echo $X
declare_vars:
executor: generic.local.bash
type: script
description: declaring variables
vars:
First: Bob
Last: Bill
run: |
echo "First:" $First
echo "Last:" $Last
declare_shell_sh:
executor: generic.local.sh
type: script
description: declare shell name to sh
shell: sh
run: hostname
declare_shell_bash:
executor: generic.local.bash
type: script
description: declare shell name to bash
shell: bash
run: hostname
declare_shell_python:
executor: generic.local.python
type: script
description: declare shell name to python
shell: python
run: |
print("Hello World")
declare_shell_bin_bash:
executor: generic.local.bash
type: script
description: declare shell name to /bin/bash
shell: "/bin/bash -e"
run: hostname
declare_shell_name_bin_sh:
executor: generic.local.sh
type: script
description: declare shell name to /bin/sh
shell: "/bin/sh -e"
run: hostname
declare_shell_opts:
executor: generic.local.sh
type: script
description: declare shell name to sh
shell: "sh -e"
run: hostname
declare_shell_bin_zsh:
executor: generic.local.zsh
type: script
description: declare shell zsh
shell: "zsh"
run: hostname
declare_shell_zsh:
executor: generic.local.zsh
type: script
description: declare shell /bin/zsh
shell: "zsh"
run: hostname
declare_shell_bin_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/csh
shell: "/bin/csh"
run: hostname
declare_shell_csh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "csh"
run: hostname
declare_shell_bin_tcsh:
executor: generic.local.csh
type: script
description: declare shell /bin/tcsh
shell: "/bin/tcsh"
run: hostname
declare_shell_tcsh:
executor: generic.local.csh
type: script
description: declare shell tcsh
shell: "tcsh"
run: hostname
declare_shebang:
executor: generic.local.bash
type: script
description: declare shell name to sh
shebang: "#!/usr/bin/env bash"
run: hostname
status_returncode_list:
executor: generic.local.bash
type: script
description: The returncode can be a list of integers
run: exit 0
status:
returncode: [0]
status_returncode_int:
executor: generic.local.bash
type: script
description: The returncode can be an integer to match with single returncode
run: exit 0
status:
returncode: 0
status_regex:
executor: generic.local.bash
type: script
description: This test pass with a regular expression status check
run: hostname
status:
regex:
stream: stdout
exp: "^$"
status_regex_returncode:
executor: generic.local.bash
type: script
description: This test fails because returncode and regex specified
run: hostname
status:
returncode: [0]
regex:
stream: stdout
exp: "^hello"
status_runtime_min_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if its within 1.0-4.0sec"
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
max: 4.0
status_runtime_min:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if exceeds mintime of 1.0sec"
tags: ["tutorials"]
run: sleep 2
status:
runtime:
min: 1.0
status_runtime_max:
type: script
executor: generic.local.sh
description: "Run a sleep job for 2 seconds and test pass if less than maxtime of 4.0sec"
tags: ["tutorials"]
run: sleep 2
status:
runtime:
max: 4.0
sbatch_example:
type: script
executor: generic.local.bash
description: This test runs hostname using sbatch directives
sbatch:
- "-t 10:00:00"
- "-p normal"
- "-N 1"
- "-n 8"
run: hostname
bsub_example:
type: script
executor: generic.local.bash
description: This test runs hostname using bsub directives
bsub:
- "-W 00:30"
- "-N 1"
run: hostname
cobalt_example:
type: script
executor: generic.local.bash
description: This test runs hostname using cobalt directives
cobalt:
- "-t 30"
- "-n 1"
run: hostname
skip_example:
type: script
executor: generic.local.bash
description: this test is skip
skip: true
run: hostname
tag_str_example:
type: script
executor: generic.local.bash
description: tags can be defined as string
tags: network
run: hostname
tag_list_example:
type: script
executor: generic.local.bash
description: This is a tag example using list
sbatch:
- "-t 10:00:00"
- "-p normal"
- "-N 1"
- "-n 8"
tags: ["slurm"]
run: hostname
run_only_example:
type: script
executor: generic.local.bash
description: run_only example that runs with user1 on Linux system (rhel, centos) with LSF
run_only:
user: user1
scheduler: lsf
platform: Linux
linux_distro:
- rhel
- centos
run: |
uname -av
lsinfo
metrics_regex_example:
type: script
executor: generic.local.bash
description: metrics regular expression example
run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
metrics:
hpcg_rating:
regex:
exp: 'rating of=(\d+\.\d+)$'
stream: stdout
metric_variable_assignment:
executor: generic.local.sh
type: script
description: capture result metric based on variables and environment variable
vars:
GFLOPS: "63.6515"
env:
FOO: BAR
run: |
echo $GFLOPS
echo $FOO
tags: tutorials
metrics:
gflops:
vars: "GFLOPS"
foo:
env: "FOO"
multi_executor_vars:
type: script
executor: 'generic.local.(sh|bash)'
description: single test multiple executor with variable declaration
run: |
echo $X
echo $Y
executors:
generic.local.sh:
vars:
X: 1
Y: 2
generic.local.bash:
vars:
X: 10
Y: 11
multi_executor_environment:
type: script
executor: 'generic.local.(sh|bash)'
description: single test multiple executor with environment declaration
run: echo $SHELL
executors:
generic.local.sh:
env:
SHELL: sh
generic.local.bash:
env:
SHELL: bash
executors_sbatch_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring sbatch by executors section
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
executors_bsub_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring bsub by executors section
tags: [tutorials]
run: hostname
executors:
generic.local.bash:
bsub: ["-n 4", "-W 30"]
generic.local.sh:
bsub: ["-n 8", "-W 60"]
executors_pbs_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring pbs by executors section
tags: [tutorials]
run: hostname
executors:
generic.local.bash:
pbs: ["-l ncpus=4", "-l walltime=30"]
generic.local.sh:
pbs: ["-l ncpus=8", "-l walltime=60"]
executors_status_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring status by executors section
tags: [tutorials]
run: exit 0
executors:
generic.local.bash:
status:
returncode: 0
generic.local.sh:
status:
returncode: [1, 2]
executors_metrics_declaration:
type: script
executor: 'generic.local.(bash|sh)'
description: Declaring metrics by executors section
tags: [tutorials]
run: echo "Hello World"
executors:
generic.local.bash:
metrics:
hello:
regex:
stream: stdout
exp: "(Hello)"
generic.local.sh:
metrics:
world:
regex:
stream: stdout
exp: "(World)"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/script-v1.0.schema.json/invalid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
invalid_test_name_&!@#$%:
type: script
executor: generic.local.bash
description: "invalid test name"
invalid_bash:
type: script
executor: generic.local.bash
shell: "bash-missing-run"
missing_run_key:
type: script
executor: generic.local.bash
description: invalid key name roon, missing run key
roon: |
systemctl is-active slurmd
systemctl is-enabled slurmd | grep enabled
invalid_env_type:
type: script
executor: generic.local.bash
description: env key should be a dictionary
env:
- FOO=BAR
run: echo $FOO
invalid_vars_type:
type: script
executor: generic.local.bash
description: var key should be a dictionary
vars:
- FOO=BAR
run: echo $FOO
invalid_description:
type: script
executor: generic.local.bash
description:
- "Multi Line description"
- "is not accepted"
invalid_regex_stream:
type: script
executor: generic.local.bash
description: This test fails because of invalid regex stream
run: hostname
status:
regex:
stream: file
exp: "world$"
regex_additionalProperties_test:
type: script
executor: generic.local.bash
description: Testing for additional properties in regex field
run: hostname
status:
regex:
stream: stdout
exp: "world$"
X: 1
missing_regex_exp:
type: script
executor: generic.local.bash
description: This test fails because of missing key 'exp' in regex
run: hostname
status:
regex:
stream: stdout
invalid_returncode_type:
type: script
executor: generic.local.bash
description: This test fails because of invalid return code type
run: hostname
status:
returncode: ["1"]
empty_returncode_list:
type: script
executor: generic.local.bash
description: An empty returncode list will cause an error
run: hostname
status:
returncode: []
non_int_returncodes:
type: script
executor: generic.local.bash
description: The returncode must be an int and not a number
run: exit 1
status:
returncode: 1.01
non_int_returncodes_list:
type: script
executor: generic.local.bash
description: The returncode must be a list of integers and no numbers
run: exit 1
status:
returncode: [1, 2.230]
invalid_shell_usr_bin_bash:
type: script
executor: generic.local.bash
description: invalid shell name '/usr/bin/bash'
shell: /usr/bin/bash
run: hostname
invalid_shell_type:
type: script
executor: generic.local.bash
description: invalid shell type must be a string
shell: ["/bin/bash"]
run: hostname
invalid_type_shell_shebang:
type: script
executor: generic.local.bash
description: invalid type for shell shebang, must be a string
shebang: ["#!/bin/bash"]
run: hostname
invalid_skip_value:
type: script
executor: generic.local.bash
description: invalid value for skip, must be boolean
skip: 1
run: hostname
empty_tags:
type: script
executor: generic.local.bash
description: tag list can't be empty, requires one item.
tags: []
run: hostname
non_unique_tags:
type: script
executor: local.bash
description: tag names must be unique
tags: ["network", "network"]
run: hostname
invalid_tags_value:
type: script
executor: generic.local.bash
description: invalid tag value must be all string items
tags: ["network", 400 ]
run: hostname
additionalProperties_test:
type: script
executor: generic.local.bash
description: additional properties are not allowed so any invalid key/value pair will result in error
FOO: BAR
run: hostname
additionalProperties_status:
type: script
executor: generic.local.bash
description: test additional properties in status object. This is not allowed
sbatch: [ "-n 2", "-q normal", "-t 10"]
run: hostname
status:
slurm_job_state: "COMPLETED"
FOO: BAR
invalid_runtime_min:
type: script
executor: generic.local.sh
description: "Invalid type for min property in runtime. Must be an integer or float not a string"
run: sleep 2
status:
runtime:
min: "1"
runtime_min_must_exceed_0:
type: script
executor: generic.local.sh
description: "The runtime must exceed 0"
run: sleep 2
status:
runtime:
min: -1
invalid_slurm_job_state:
type: script
executor: generic.local.sh
description: invalid value for slurm_job_state, should raise error with enum values.
sbatch:
- "-n 2"
- "-q normal"
- "-t 10"
run: hostname
status:
slurm_job_state: "FINISH"
duplicate_linux_distro:
type: script
executor: generic.local.bash
description: Duplicate items in linux_distro is not allowed
run_only:
linux_distro:
- rhel
- rhel
run: uname -av
empty_list_linux_distro:
type: script
executor: generic.local.bash
description: Empty List in linux_distro is not allowed. Requires atleast 1 item
run_only:
linux_distro: []
run: uname -av
additionalProperties_run_only:
type: script
executor: generic.local.bash
description: additional Properties not allowed in run_only field. Invalid field python
run_only:
user: root
python: 3.5
run: hostname
invalid_metrics_additional_property:
type: script
executor: generic.local.bash
description: Test for additional property for metrics property
vars:
FOO: BAR
run: echo $FOO
metrics:
foo:
variable: FOO
invalid_metrics_type:
type: script
executor: generic.local.bash
description: metrics property is an object, testing for type
vars:
FOO: BAR
run: echo $FOO
metrics: FOO
executors_invalid_var_type:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Invalid type field for 'vars'
tags: [tutorials]
run: echo $FOO
executors:
generic.local.bash:
vars: ["FOO=BAR"]
executors_additionalProperties:
type: script
executor: "generic.local.(bash|sh|zsh)"
description: Testing for additional properties in 'executors'
tags: [tutorials]
run: hostname
sbatch: ["-N 4"]
executors:
generic.local.bash:
sbatch: ["-n 4", "-N 1", "-t 30"]
FOO: BAR
generic.local.sh:
sbatch: ["-n 8", "-N 1", "-t 60"]
generic.local.zsh:
sbatch: ["-n 16", "-N 2", "-t 120"]
Compiler Schema Examples¶
$ buildtest schema -n compiler-v1.0.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/compiler-v1.0.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
gnu_example:
executor: local.bash
type: compiler
description: "gnu example with modules, and cflags example"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc@8.4.0:
cflags: "-O3"
intel_example:
executor: local.bash
type: compiler
description: "intel example using cflags"
source: src/hello.c
compilers:
name: [intel]
config:
intel@2018:
cflags: "-O1"
clang_example:
executor: local.bash
type: compiler
description: "clang example using cflags"
source: src/hello.c
compilers:
name: [clang]
default:
clang:
cflags: "-O1"
config:
clang@11:
cflags: "-O2"
upcxx_example:
executor: local.bash
type: compiler
description: "upcxx compiler declaration in default and config section "
source: src/hello.c
compilers:
name: [upcxx]
default:
upcxx:
cflags: "-g aries"
config:
upcxx@2020:
cflags: "-O1 -g aries"
pgi_example:
executor: local.bash
type: compiler
description: "pgi example using cxxflags, ldflags in default and config section"
source: src/hello.cpp
compilers:
name: ["^(pgi|PrgEnv)"]
default:
pgi:
cxxflags: "-O1"
ldflags: "-lm"
config:
pgi@18.1:
module:
swap: [PrgEnv-gnu, PrgEnv-pgi]
load: [pgi/18.1]
pgi@18.2:
module:
swap: [PrgEnv-gnu, PrgEnv-pgi]
load: [pgi/18.2]
cray_example:
executor: local.bash
type: compiler
description: "cray example using fflags and cppflags"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
cray:
fflags: "-O1"
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
sbatch_example_all_compiler_groups:
type: compiler
description: sbatch example to for all compiler groups
executor: local.bash
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
cray:
fflags: "-O1"
all:
sbatch: ["-t 10", "-n 2", "-C haswell" ]
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
bsub_all_compiler_groups:
type: compiler
description: bsub example for all compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
bsub: ["-W 00:30", "-n 2"]
config:
intel@2019:
cxxflags: "-O1"
cobalt_all_compiler_groups:
type: compiler
description: cobalt example for all compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
cobalt: ["-t 30", "-n 1"]
config:
intel@2019:
cxxflags: "-O1"
sbatch_compiler_group:
type: compiler
description: sbatch example in multiple compiler groups.
executor: local.bash
source: src/hello.f90
compilers:
name: ["^(gcc|intel)"]
default:
gcc:
fflags: "-O1"
sbatch: ["-t 10", "-n 2", "-C haswell" ]
intel:
fflags: "-O2"
sbatch: ["-t 10", "-n 2", "-C knl" ]
config:
gcc@8.1.0:
sbatch: ["-t 60", "-n 2", "-C knl"]
module:
swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]
bsub_compiler_group:
type: compiler
description: bsub example in multiple compiler groups.
executor: local.bash
source: src/hello.f90
compilers:
name: ["^(gcc|intel)"]
default:
gcc:
fflags: "-O1"
bsub: ["-W 00:30", "-n 2" ]
intel:
fflags: "-O2"
bsub: ["-W 00:30", "-n 4" ]
config:
gcc@8.1.0:
bsub: ["-W 00:30", "-n 6" ]
module:
swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]
env_example:
type: compiler
description: Setting environment variables
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
env:
OMP_NUM_THREADS: 2
run: $_EXEC 1 2 4
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
custom_env_by_compiler_group:
type: compiler
description: Setting environment variables in compiler groups
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc|pgi)"]
default:
all:
run: $_EXEC 1 2 4
gcc:
cxxflags: "-fopenmp"
env:
OMP_NUM_THREADS: 4
pgi:
cxxflags: "-mp"
env:
OMP_NUM_THREADS: 6
config:
gcc@10.2.0:
env:
OMP_NUM_THREADS: 6
gcc@9.2.0:
env:
OMP_NUM_THREADS: 8
pgi@9.2.0:
env:
OMP_NUM_THREADS: 10
vars_example:
type: compiler
description: Setting shell variables
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
vars:
OUTFILE: /tmp/file1.txt
run: $_EXEC > $OUTFILE
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
pass_args_run:
type: compiler
description: Passing arguments to executable in run section
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [intel]
default:
all:
run: $_EXEC 1 2 4
config:
intel@2019:
cxxflags: "-O1"
mpi_launcher_example:
type: compiler
description: mpi launcher example
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [gcc]
default:
all:
run: mpirun -np 2 $_EXEC
config:
gcc@7.3.0:
cflags: "-O3"
cxx: mpicxx
status_returncode_example:
type: compiler
description: Status returncode match example
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [gnu]
default:
all:
vars:
OUTFILE: /tmp/file1.txt
run: $_EXEC > $OUTFILE
status:
returncode: 1
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
pre_post_build_run_sections:
type: compiler
description: Run commands pre and post build section
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc)"]
default:
all:
pre_build: echo "pre-build section for ALL compilers"
post_build: echo "post-build section for ALL Compilers"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
gcc:
pre_build: echo "pre-build section for GCC compilers"
post_build: echo "post-build section for GCC compilers"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
config:
gcc@7.3.0:
pre_build: echo "pre-build section for gcc@7.3.0"
post_build: echo "post-build section for gcc@7.3.0"
pre_run: echo "pre-run section for ALL compilers"
post_run: echo "post-run section for ALL Compilers"
cflags: "-O3"
gcc@8.2.0:
pre_build: echo "gcc --version"
cflags: "-O3"
multi_compilers:
type: compiler
description: Select one or more compilers to run test
executor: local.bash
source: "src/hello.cpp"
compilers:
name: ["^(gcc|intel|pgi|cray)"]
exclude: [intel@18]
default:
gcc:
cflags: "-fopenmp"
intel:
cflags: "-qopenmp"
pgi:
cflags: "-fopenmp"
cray:
cflags: "-h omp"
config:
gcc@7.5.0:
cflags: "-O3"
module:
load: [gcc/7.5.0]
intel@17:
module:
load: [intel/2017]
intel@18:
module:
load: [intel/2018]
pgi/18.0:
module:
load: [pgi/18.0]
craype/2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray]
load: [craype/2.6.2]
metrics_example:
type: compiler
description: Recording test metrics with compiler schema
executor: local.bash
source: "src/hello.cpp"
compilers:
name: [gnu]
default:
all:
vars:
OUTFILE: /tmp/file1.txt
run: $_EXEC > $OUTFILE
status:
returncode: 1
config:
gcc@10.2.0:
cxxflags: "-fopenmp"
metrics:
outfile:
vars: "OUTFILE"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/compiler-v1.0.schema.json/invalid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
missing_type:
executor: local.bash
description: "type key is missing, this is a required field"
source: "src/hello.c"
compilers:
name: [intel]
missing_required_compilers:
executor: local.bash
type: compiler
description: "missing required field compilers "
source: "src/hello.c"
missing_required_source:
executor: local.bash
type: compiler
description: "missing required field 'source' "
compilers:
name: [gcc]
invalid_type_value:
executor: local.bash
type: script
description: "invalid value for type field must be 'compiler' "
source: src/hello.c
compilers:
name: [gcc]
invalid_description_value:
executor: local.bash
type: compiler
description: 1
source: src/hello.c
compilers:
name: [gcc]
invalid_type_module:
executor: local.bash
type: compiler
description: "type for 'module' key, expecting a property but received 'string' "
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module: "module load gcc/9.2.0"
module_purge_invalid_type:
executor: local.bash
type: compiler
description: "The purge property module is invalid. Expects bool got an int"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
purge: 1
module_swap_duplicate_check:
executor: local.bash
type: compiler
description: "The swap property expects two unique items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0, gcc/8.0]
module_swap_min_items:
executor: local.bash
type: compiler
description: "The swap property expects a minimum of 2 items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0]
module_swap_max_items:
executor: local.bash
type: compiler
description: "The swap property expects a maximum of 2 items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
swap: [gcc/8.0, gcc/9.0, gcc/10.0]
module_load_duplicate_items:
executor: local.bash
type: compiler
description: "The load property expects unique items"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
load: [gcc/9.2.0, gcc/9.2.0]
module_load_min_items:
executor: local.bash
type: compiler
description: "The load property expects a minimum of 1 item"
source: src/hello.c
compilers:
name: [gcc]
config:
gcc/9.2.0:
module:
load: []
additionalProperties_main:
executor: local.bash
type: compiler
description: "test additionalProperties in main schema"
foo: bar
source: src/hello.c
compilers:
name: [gcc]
missing_required_compiler_name:
executor: local.bash
type: compiler
description: "'name' field in compilers section is required field"
source: src/hello.f90
compilers:
default:
cray:
fflags: "-O1"
config:
PrgEnv-cray@2.6.2:
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
uniqueItems_compiler_name:
executor: local.bash
type: compiler
description: "Test unique items in 'name' field in compilers section"
source: src/hello.f90
compilers:
name: ["^(PrgEnv-cray)", "^(PrgEnv-cray)"]
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler section"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
FOO: BAR
default:
all:
env:
X: 1
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler_default_all:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler default all section"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
default:
all:
XYZ: 123
config:
PrgEnv-cray@2.6.2:
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
additionalProperties_compiler_config:
executor: local.bash
type: compiler
description: "Test additionalProperies in compiler config section, FOO: BAR"
source: src/hello.f90
compilers:
name: ["PrgEnv-cray"]
config:
PrgEnv-cray@2.6.2:
FOO: BAR
fflags: "-O1"
module:
swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]
Spack Schema Examples¶
$ buildtest schema -n spack-v1.0.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/spack-v1.0.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
env_create_name:
type: spack
executor: generic.local.sh
description: Create spack environment by name
spack:
root: $HOME/spack/
env:
create:
name: myproject
specs:
- m4
- zlib
install:
options: ''
env_activate:
type: spack
executor: generic.local.sh
description: Activate spack environment by name
spack:
root: $HOME/spack/
env:
activate:
name: myproject
specs:
- m4
- zlib
install:
options: ''
env_create_directory:
type: spack
executor: generic.local.sh
description: Create spack environment by directory
spack:
root: $HOME/spack/
env:
create:
dir: $HOME/spack-env/myproject
specs:
- 'm4'
- 'zlib@1.2.11'
install:
options: '--cache-only'
env_create_from_manifest:
type: spack
executor: generic.local.sh
description: Create spack enviromment from manifest file
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
env_concretized_install:
type: spack
description: run 'spack concretize -f' in an environment and install specs
executor: generic.local.sh
spack:
root: $HOME/spack/
env:
create:
name: myproject
manifest: $HOME/spack.yaml
concretize: true
install:
options: '--cache-only'
env_mirror:
type: spack
executor: generic.local.sh
description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in environment
spack:
root: $HOME/spack/
env:
mirror:
h5: /path/to/mirror
create:
name: myproject
manifest: $HOME/spack.yaml
install:
options: '--cache-only'
install_without_env:
type: spack
executor: generic.local.sh
description: install specs without environment
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
pre_post_cmd_spack_install:
type: spack
executor: generic.local.sh
description: run commands before and after spack using pre_cmds and post_cmds
pre_cmds: |
cd $HOME
git clone https://github.com/spack/spack
spack:
root: $HOME/spack/
install:
options: '--cache-only'
specs: ['m4', 'bzip2']
post_cmds: |
spack find
remove_spack_environment:
type: spack
executor: generic.local.sh
description: remove spack environment explicitly before creating environment
spack:
root: $HOME/spack
env:
rm:
name: m4
create:
name: m4
activate:
name: m4
specs:
- 'm4'
remove_spack_environment_automatically:
type: spack
executor: generic.local.sh
description: remove spack environment automatically
spack:
root: $HOME/spack
env:
create:
remove_environment: true
name: m4
activate:
name: m4
specs:
- 'm4'
spack_test_results_specs_suites:
type: spack
executor: generic.local.sh
description: "Report results using suitename and spec format in spack test results"
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib' ]
test:
remove_tests: true
run:
specs: ['m4', 'zlib']
results:
option: '-f'
suite: ['zlib']
specs: ['m4']
post_cmds: |
spack find
rm -rf $SPACK_ROOT
sbatch_field_with_spack:
type: spack
executor: generic.local.sh
description: Specify sbatch field with spack schema
sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"]
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
skip_test_in_spack:
skip: True
type: spack
executor: generic.local.sh
description: This test will be skipped
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
var_declaration_in_spack:
type: spack
executor: generic.local.sh
description: Define variables and environment variables in spack
vars:
FOO: "BAR"
env:
SPACK_GNUPGHOME: "$HOME/.gnupg"
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
metrics_example_vars_envs:
type: spack
executor: generic.local.sh
description: Define metrics variable and environment variable in spack
vars:
FOO: "BAR"
env:
SPACK_GNUPGHOME: "$HOME/.gnupg"
metrics:
foo:
vars: "FOO"
spack_gpg:
env: "SPACK_GNUPGHOME"
spack:
root: $HOME/spack
install:
specs: ['m4']
options: "-v"
spack_sbatch_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "sbatch directives can be defined in spack schema"
tags: [spack]
executors:
generic.local.sh:
sbatch: ["-N 1", "-t 20"]
generic.local.bash:
sbatch: ["-N 8", "-t 10"]
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true
vars_multi_executors:
type: spack
executor: "generic.local.(sh|bash)"
description: "variable declaration with multiple executors"
tags: [spack]
executors:
generic.local.sh:
vars:
FOO: BAR
generic.local.bash:
vars:
HELLO: WORLD
spack:
root: $HOME/spack
env:
specs:
- 'm4'
activate:
name: m4
concretize: true
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.11.0/buildtest/schemas/examples/spack-v1.0.schema.json/invalid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
missing_root_to_spack:
type: spack
executor: generic.local.sh
description: 'root property required to run source $SPACK_ROOT/share/spack/setup-env.sh'
spack:
env:
create:
name: myproject
specs:
- m4
- zlib
specs_must_be_list_of_strings:
type: spack
executor: generic.local.sh
description: 'specs must be a list of strings '
spack:
root: $HOME/spack
env:
create:
name: myproject
specs:
- 1
- zlib
additionalProperties_spack_field:
type: spack
executor: generic.local.sh
description: additional Properties can't be specified in spack section
spack:
root: $HOME/spack
FOO: BAR
env:
create:
name: myproject
specs:
- zlib
invalid_type_mirror_field:
type: spack
executor: generic.local.sh
description: The mirror field must be a key value pair
spack:
root: $HOME/spack
mirror: https://caches.e4s.io
env:
create:
name: myproject
specs:
- zlib
spack_test_additionalProperties:
type: spack
executor: generic.local.sh
description: "Check for additionalProperties in test section. FOO key is not allowed"
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib' ]
test:
FOO: BAR
remove_tests: true
run:
specs: ['m4', 'zlib']
spack_test_results_testing_required_oneOf:
type: spack
executor: generic.local.sh
description: "The results property expects one to specify specs, suite, or both"
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib' ]
test:
remove_tests: true
run:
specs: ['m4', 'zlib']
results:
option: '-f'
post_cmds: |
spack find
rm -rf $SPACK_ROOT
spack_test_run_invalid_spec:
type: spack
executor: generic.local.sh
description: "specs property requires a list of strings. "
tags: [spack]
pre_cmds: |
cd /tmp
git clone https://github.com/spack/spack
spack:
root: /tmp/spack
verify_spack: false
install:
specs: ['m4', 'zlib' ]
test:
remove_tests: true
run:
specs: ['m4', 1]
results:
option: '-f'
suite: ['zlib']
specs: ['m4']
post_cmds: |
spack find
rm -rf $SPACK_ROOT