Buildtest Schemas

Schema Naming Convention

All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.

Schema Examples

The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.

In order to view an example you can run:

buildtest schema -n <schema> --example

Definition Schema

This schema is used for declaring definitions that need to be reused in multiple schemas. We use $ref keyword to reference definitions from this file.

Schema Content

$ buildtest schema -n definitions.schema.json --json
{
  "$id": "definitions.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "JSON Schema Definitions File. ",
  "description": "This file is used for declaring definitions that are referenced from other schemas",
  "definitions": {
    "list_of_strings": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
    "string_or_list": {
      "oneOf": [
        {
          "type": "string"
        },
        {
          "$ref": "#/definitions/list_of_strings"
        }
      ]
    },
    "list_of_ints": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "integer"
      }
    },
    "int_or_list": {
      "oneOf": [
        {
          "type": "integer"
        },
        {
          "$ref": "#/definitions/list_of_ints"
        }
      ]
    },
    "regex": {
      "type": "object",
      "description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
      "required": [
        "stream",
        "exp"
      ],
      "additionalProperties": false,
      "properties": {
        "stream": {
          "type": "string",
          "enum": [
            "stdout",
            "stderr"
          ],
          "description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
        },
        "exp": {
          "type": "string",
          "description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
        }
      }
    },
    "env": {
      "type": "object",
      "description": "One or more key value pairs for an environment (key=value)",
      "minItems": 1,
      "items": {
        "type": "object",
        "minItems": 1,
        "propertyNames": {
          "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
        }
      }
    },
    "description": {
      "type": "string",
      "description": "The ``description`` field is used to document what the test is doing",
      "maxLength": 80
    },
    "tags": {
      "description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
      "$ref": "#/definitions/string_or_list"
    },
    "skip": {
      "type": "boolean",
      "description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
    },
    "executor": {
      "type": "string",
      "description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
    },
    "metrics_field": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "regex": {
          "$ref": "#/definitions/regex"
        },
        "vars": {
          "type": "string",
          "description": "Assign value to metric based on variable name"
        },
        "env": {
          "type": "string",
          "description": "Assign value to metric based on environment variable"
        }
      }
    },
    "metrics": {
      "type": "object",
      "description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
      "patternProperties": {
        "^.*$": {
          "$ref": "#/definitions/metrics_field",
          "description": "Name of metric"
        }
      }
    },
    "run_only": {
      "type": "object",
      "description": "A set of conditions to specify when running tests. All conditions must pass in order to process test.",
      "additionalProperties": false,
      "properties": {
        "scheduler": {
          "type": "string",
          "description": "Test will run only if scheduler is available. We assume binaries are available in $PATH",
          "enum": [
            "lsf",
            "slurm",
            "cobalt",
            "pbs"
          ]
        },
        "user": {
          "type": "string",
          "description": "Test will run only if current user matches this field, otherwise test will be skipped"
        },
        "platform": {
          "type": "string",
          "description": "This test will run if target system is Linux or Darwin. We check target system using ``platform.system()`` and match with input field",
          "enum": [
            "Linux",
            "Darwin"
          ]
        },
        "linux_distro": {
          "type": "array",
          "description": "Specify a list of Linux Distros to check when processing test. If target system matches one of input field, test will be processed.",
          "uniqueItems": true,
          "minItems": 1,
          "items": {
            "type": "string",
            "enum": [
              "darwin",
              "ubuntu",
              "debian",
              "rhel",
              "centos",
              "fedora",
              "sles",
              "opensuse",
              "amazon",
              "arch"
            ]
          }
        }
      }
    },
    "batch": {
      "type": "object",
      "description": "The ``batch`` field is used to specify scheduler agnostic directives that are translated to #SBATCH or #BSUB based on your scheduler. This is an experimental feature that supports a subset of scheduler parameters.",
      "additionalProperties": false,
      "properties": {
        "account": {
          "type": "string",
          "description": "Specify Account to charge job"
        },
        "begintime": {
          "type": "string",
          "description": "Specify begin time when job will start allocation"
        },
        "cpucount": {
          "type": "string",
          "description": "Specify number of CPU to allocate"
        },
        "email-address": {
          "type": "string",
          "description": "Email Address to notify on Job State Changes"
        },
        "exclusive": {
          "type": "boolean",
          "description": "Specify if job needs to run in exclusive mode"
        },
        "memory": {
          "type": "string",
          "description": "Specify Memory Size for Job"
        },
        "network": {
          "type": "string",
          "description": "Specify network resource requirement for job"
        },
        "nodecount": {
          "type": "string",
          "description": "Specify number of Nodes to allocate"
        },
        "qos": {
          "type": "string",
          "description": "Specify Quality of Service (QOS)"
        },
        "queue": {
          "type": "string",
          "description": "Specify Job Queue"
        },
        "tasks-per-core": {
          "type": "string",
          "description": "Request number of tasks to be invoked on each core. "
        },
        "tasks-per-node": {
          "type": "string",
          "description": "Request number of tasks to be invoked on each node. "
        },
        "tasks-per-socket": {
          "type": "string",
          "description": "Request the maximum tasks to be invoked on each socket. "
        },
        "timelimit": {
          "type": "string",
          "description": "Specify Job timelimit"
        }
      }
    },
    "status": {
      "type": "object",
      "description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
      "additionalProperties": false,
      "properties": {
        "slurm_job_state": {
          "type": "string",
          "enum": [
            "COMPLETED",
            "FAILED",
            "OUT_OF_MEMORY",
            "TIMEOUT"
          ],
          "description": "This field can be used for checking Slurm Job State, if there is a match buildtest will report as ``PASS`` "
        },
        "returncode": {
          "description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
          "$ref": "#/definitions/int_or_list"
        },
        "regex": {
          "$ref": "#/definitions/regex",
          "description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
        },
        "runtime": {
          "type": "object",
          "description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
          "properties": {
            "min": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
            },
            "max": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
            }
          }
        }
      }
    },
    "BB": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Create burst buffer space, this specifies #BB options in your test."
    },
    "DW": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Specify Data Warp option (#DW) when using burst buffer."
    },
    "sbatch": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #SBATCH options in test script."
    },
    "bsub": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #BSUB options in test script."
    },
    "cobalt": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #COBALT options in test script."
    },
    "pbs": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #PBS directives in test script."
    },
    "executors": {
      "type": "object",
      "description": "Define executor specific configuration",
      "patternProperties": {
        "description": "Name of executor to override configuration",
        "^.*$": {
          "additionalProperties": false,
          "properties": {
            "env": {
              "$ref": "#/definitions/env"
            },
            "vars": {
              "$ref": "#/definitions/env"
            },
            "sbatch": {
              "$ref": "#/definitions/list_of_strings"
            },
            "bsub": {
              "$ref": "#/definitions/list_of_strings"
            },
            "pbs": {
              "$ref": "#/definitions/list_of_strings"
            },
            "cobalt": {
              "$ref": "#/definitions/list_of_strings"
            },
            "BB": {
              "$ref": "#/definitions/BB"
            },
            "DW": {
              "$ref": "#/definitions/DW"
            },
            "status": {
              "$ref": "#/definitions/status"
            },
            "metrics": {
              "$ref": "#/definitions/metrics"
            }
          }
        }
      }
    }
  }
}

Settings Schema

This schema defines how buildtest configuration file is validated.

Schema Content

$ buildtest schema -n settings.schema.json --json
{
  "$id": "settings.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "buildtest configuration schema",
  "type": "object",
  "required": [
    "system"
  ],
  "additionalProperties": false,
  "properties": {
    "system": {
      "type": "object",
      "patternProperties": {
        "^.*$": {
          "$ref": "#/definitions/system"
        }
      }
    }
  },
  "definitions": {
    "system": {
      "required": [
        "executors",
        "moduletool",
        "load_default_buildspecs",
        "hostnames",
        "compilers"
      ],
      "additionalProperties": false,
      "type": "object",
      "properties": {
        "hostnames": {
          "type": "array",
          "description": "Specify a list of hostnames to check where buildtest can run for the given system record",
          "items": {
            "type": "string"
          }
        },
        "description": {
          "type": "string",
          "description": "system description field"
        },
        "buildspec_roots": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
        },
        "load_default_buildspecs": {
          "type": "boolean",
          "description": "Specify whether buildtest should automatically load  buildspecs provided in buildtest repo into buildspec cache"
        },
        "testdir": {
          "type": "string",
          "description": "Specify full path to test directory where buildtest will write tests."
        },
        "logdir": {
          "type": "string",
          "description": "Specify location where buildtest will write log files"
        },
        "moduletool": {
          "type": "string",
          "description": "Specify modules tool used for interacting with ``module`` command. ",
          "enum": [
            "environment-modules",
            "lmod",
            "N/A"
          ]
        },
        "processor": {
          "type": "object",
          "description": "Specify processor information",
          "additionalProperties": false,
          "properties": {
            "numcpus": {
              "type": "integer",
              "minimum": 1,
              "description": "Specify Total Number of CPUs"
            },
            "sockets": {
              "type": "integer",
              "minimum": 1,
              "description": "Specify Number of CPU Sockets"
            },
            "cores": {
              "type": "integer",
              "minimum": 1,
              "description": "Specify Number of Physical Cores"
            },
            "threads_per_core": {
              "type": "integer",
              "minimum": 1,
              "description": "Specify Threads per Core"
            },
            "core_per_socket": {
              "type": "integer",
              "minimum": 1,
              "description": "Specify Cores per Socket"
            },
            "model": {
              "type": "string",
              "description": "Specify Processor Model"
            },
            "arch": {
              "type": "string",
              "description": "Specify processor architecture"
            },
            "vendor": {
              "type": "string",
              "description": "Vendor Name"
            }
          }
        },
        "compilers": {
          "type": "object",
          "description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
          "additionalProperties": false,
          "properties": {
            "find": {
              "type": "object",
              "additionalProperties": false,
              "description": "Find compilers by specifying regular expression that is applied to modulefile names",
              "properties": {
                "gcc": {
                  "type": "string",
                  "description": "Specify a regular expression to search for gcc compilers from your module stack"
                },
                "intel": {
                  "type": "string",
                  "description": "Specify a regular expression to search for intel compilers from your module stack"
                },
                "cray": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cray compilers from your module stack"
                },
                "clang": {
                  "type": "string",
                  "description": "Specify a regular expression to search for clang compilers from your module stack"
                },
                "cuda": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cuda compilers from your module stack"
                },
                "pgi": {
                  "type": "string",
                  "description": "Specify a regular expression to search for pgi compilers from your module stack"
                },
                "upcxx": {
                  "type": "string",
                  "description": "Specify a regular expression to search for upcxx compilers from your module stack"
                }
              }
            },
            "compiler": {
              "type": "object",
              "additionalProperties": false,
              "description": "Start of compiler declaration",
              "properties": {
                "gcc": {
                  "description": "Declaration of one or more GNU compilers where we define C, C++ and Fortran compiler. The GNU compiler wrapper are ``gcc``, ``g++`` and ``gfortran``. ",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "intel": {
                  "description": "Declaration of one or more Intel compilers where we define C, C++ and Fortran compiler. The Intel compiler wrapper are ``icc``, ``icpc`` and ``ifort``. ",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "cray": {
                  "description": "Declaration of one or more Cray compilers where we define C, C++ and Fortran compiler. The Cray compiler wrapper are ``cc``, ``CC`` and ``ftn``.",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "pgi": {
                  "description": "Declaration of one or more PGI compilers where we define C, C++ and Fortran compiler. The PGI compiler wrapper are ``pgcc``, ``pgc++`` and ``pgfortran``.",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "clang": {
                  "description": "Declaration of one or more Clang compilers where we define C, C++ compiler. The Clang compiler wrapper are ``clang``, ``clang++``.",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "cuda": {
                  "description": "Declaration of one or more Cuda compilers where we define C compiler. The Cuda compiler wrapper is ``nvcc``. ",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                },
                "upcxx": {
                  "description": "Declaration of one or more UPCXX compilers where we define C, C++ compiler. The UPCXX compiler wrapper are ``upcxx``.",
                  "type": "object",
                  "patternProperties": {
                    "^.*$": {
                      "$ref": "#/definitions/compiler_section"
                    }
                  }
                }
              }
            }
          }
        },
        "executors": {
          "type": "object",
          "additionalProperties": false,
          "description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``, ``cobalt``. The executors are referenced in buildspec using ``executor`` field.",
          "properties": {
            "defaults": {
              "type": "object",
              "description": "Specify default executor settings for all executors",
              "additionalProperties": false,
              "properties": {
                "pollinterval": {
                  "type": "integer",
                  "description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
                  "minimum": 10,
                  "maximum": 300,
                  "default": 30
                },
                "launcher": {
                  "type": "string",
                  "enum": [
                    "sbatch",
                    "bsub",
                    "qsub"
                  ],
                  "description": "Specify batch launcher to use when submitting jobs, this is applicable for LSF and Slurm Executors."
                },
                "max_pend_time": {
                  "$ref": "#/definitions/max_pend_time"
                },
                "account": {
                  "$ref": "#/definitions/account"
                }
              }
            },
            "local": {
              "type": "object",
              "description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
              "patternProperties": {
                "^.*$": {
                  "$ref": "#/definitions/local"
                }
              }
            },
            "lsf": {
              "type": "object",
              "description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
              "patternProperties": {
                "^.*$": {
                  "$ref": "#/definitions/lsf"
                }
              }
            },
            "slurm": {
              "type": "object",
              "description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
              "patternProperties": {
                "^.*$": {
                  "$ref": "#/definitions/slurm"
                }
              }
            },
            "cobalt": {
              "type": "object",
              "description": "The ``cobalt`` section is used for declaring Cobalt executors for running jobs using Cobalt scheduler",
              "patternProperties": {
                "^.*$": {
                  "$ref": "#/definitions/cobalt"
                }
              }
            },
            "pbs": {
              "type": "object",
              "description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
              "patternProperties": {
                "^.*$": {
                  "$ref": "#/definitions/pbs"
                }
              }
            }
          }
        },
        "cdash": {
          "type": "object",
          "description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
          "required": [
            "url",
            "project",
            "site"
          ],
          "properties": {
            "url": {
              "type": "string",
              "description": "Url to CDASH server"
            },
            "project": {
              "type": "string",
              "description": "Name of CDASH project"
            },
            "site": {
              "type": "string",
              "description": "Site Name reported in CDASH"
            }
          }
        }
      }
    },
    "cc": {
      "description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
      "type": "string"
    },
    "cxx": {
      "type": "string",
      "description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "fc": {
      "type": "string",
      "description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "compiler_section": {
      "description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
      "type": "object",
      "additionalProperties": false,
      "required": [
        "cc",
        "cxx",
        "fc"
      ],
      "properties": {
        "cc": {
          "$ref": "#/definitions/cc"
        },
        "cxx": {
          "$ref": "#/definitions/cxx"
        },
        "fc": {
          "$ref": "#/definitions/fc"
        },
        "module": {
          "$ref": "#/definitions/module"
        }
      }
    },
    "unique_string_array": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string"
      }
    },
    "module": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "purge": {
          "type": "boolean",
          "description": "Run ``module purge`` if purge is set"
        },
        "load": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Load one or more modules via ``module load``"
        },
        "swap": {
          "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
          "type": "array",
          "uniqueItems": true,
          "minItems": 2,
          "maxItems": 2,
          "items": {
            "type": "string"
          }
        }
      }
    },
    "script": {
      "type": "array",
      "additionalProperties": false,
      "items": {
        "type": "string"
      }
    },
    "max_pend_time": {
      "type": "integer",
      "description": "Cancel job if it is still pending in queue beyond max_pend_time",
      "minimum": 10,
      "default": 90
    },
    "account": {
      "type": "string",
      "description": "Specify Job Account for charging resources"
    },
    "local": {
      "type": "object",
      "description": "An instance object of local executor",
      "additionalProperties": false,
      "required": [
        "shell"
      ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "shell": {
          "type": "string",
          "description": "Specify the shell launcher you want to use when running tests locally",
          "pattern": "^(/bin/bash|/bin/sh|/bin/csh|/bin/tcsh|/bin/zsh|sh|bash|csh|tcsh|zsh|python).*"
        },
        "before_script": {
          "#ref": "#/definitions/script"
        }
      }
    },
    "slurm": {
      "type": "object",
      "additionalProperties": false,
      "description": "An instance object of slurm executor",
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [
            "sbatch"
          ],
          "description": "Specify the slurm batch scheduler to use. This overrides the default ``launcher`` field. This must be ``sbatch``. "
        },
        "options": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
        },
        "cluster": {
          "type": "string",
          "description": "Specify the slurm cluster you want to use ``-M <cluster>``"
        },
        "partition": {
          "type": "string",
          "description": "Specify the slurm partition you want to use ``-p <partition>``"
        },
        "qos": {
          "type": "string",
          "description": "Specify the slurm qos you want to use ``-q <qos>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "after_script": {
          "description": "The ``after_script`` section can be used to specify commands at end of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "max_pend_time": {
          "description": "overrides default ``max_pend_time`` value",
          "$ref": "#/definitions/max_pend_time"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        }
      }
    },
    "lsf": {
      "type": "object",
      "description": "An instance object of lsf executor",
      "additionalProperties": false,
      "required": [
        "queue"
      ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [
            "bsub"
          ],
          "description": "Specify the lsf batch scheduler to use. This overrides the default ``launcher`` field. It must be ``bsub``. "
        },
        "options": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "after_script": {
          "description": "The ``after_script`` section can be used to specify commands at end of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "max_pend_time": {
          "description": "overrides default ``max_pend_time`` value",
          "$ref": "#/definitions/max_pend_time"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        }
      }
    },
    "cobalt": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [
        "queue"
      ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [
            "qsub"
          ],
          "description": "Specify the cobalt batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "after_script": {
          "description": "The ``after_script`` section can be used to specify commands at end of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "max_pend_time": {
          "description": "overrides default ``max_pend_time`` value",
          "$ref": "#/definitions/max_pend_time"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        }
      }
    },
    "pbs": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [
        "queue"
      ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [
            "qsub"
          ],
          "description": "Specify the pbs batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "after_script": {
          "description": "The ``after_script`` section can be used to specify commands at end of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "max_pend_time": {
          "description": "overrides default ``max_pend_time`` value",
          "$ref": "#/definitions/max_pend_time"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        }
      }
    }
  }
}

Schema Examples

$ buildtest schema -n settings.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/slurm-example.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    moduletool: lmod
    load_default_buildspecs: True
    buildspec_roots:
      - $HOME/buildtest-cori
    testdir: /tmp/buildtest
    executors:
      defaults:
        pollinterval: 20
        launcher: sbatch
        max_pend_time: 30
        account: admin
      slurm:
        normal:
          options: ["-C haswell"]
          qos: normal
          before_script: |
            time
            echo "commands run before job"

    compilers:
      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/cobalt-example.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    moduletool: lmod
    load_default_buildspecs: True
    executors:
      defaults:
         launcher: qsub
         max_pend_time: 30

      cobalt:
        knl:
          queue: knl

        haswell:
          queue: haswell

    compilers:
      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/pbs-example.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    moduletool: N/A
    load_default_buildspecs: True
    executors:
      defaults:
         pollinterval: 10
         launcher: qsub
         max_pend_time: 30
      pbs:
        workq:
          queue: workq
    compilers:
      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/lsf-example.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    moduletool: lmod
    load_default_buildspecs: False
    executors:
      defaults:
        pollinterval: 10
        launcher: bsub
        max_pend_time: 45
      lsf:
        batch:
          description: "LSF Executor name 'batch' that submits jobs to 'batch' queue"
          queue: batch
          account: developer
          options: ["-W 20"]
          before_script: |
            time
            echo "commands run before job"
        test:
          description: "LSF Executor name 'test' that submits jobs to 'test' queue"
          launcher: bsub
          queue: test
          account: qa
          options: ["-W 20"]
    compilers:
      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    logdir: $BUILDTEST_ROOT/logs
    testdir: $BUILDTEST_ROOT/tests

    moduletool: N/A
    load_default_buildspecs: False
    cdash:
      url: https://my.cdash.org
      project: buildtest
      site: laptop
    processor:
      numcpus: 8
      cores: 4
      threads_per_core: 2
      sockets: 1
      model: "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz"
    executors:
      local:
        bash:
          description: submit jobs on local machine using bash shell
          shell: bash
          before_script: |
            time
            echo "commands run before job"

        sh:
          description: submit jobs on local machine using sh shell
          shell: sh

        csh:
          description: submit jobs on local machine using csh shell
          shell: csh -x

        tcsh:
          description: submit jobs on local machine using tcsh shell
          shell: /bin/tcsh

        zsh:
          description: submit jobs on local machine using zsh shell
          shell: /bin/zsh

        python:
          description: submit jobs on local machine using python shell
          shell: python

    compilers:
      find:
        gcc: "^(gcc|GCC|PrgEnv-gnu)"
        intel: "^(intel|Intel|PrgEnv-intel)"
        cray: "^(cray|PrgEnv-cray)"
        clang: "^(clang|Clang)"
        cuda: "^(cuda|CUDA)"
        pgi: "^(pgi|PGI|PrgEnv-pgi)"

      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran
          gcc@7.2.0:
            cc: 'cc'
            cxx: 'cxx'
            fc: 'fc'
            module:
              load:
              - gcc/7.2.0
        intel:
          intel@2019:
            cc: 'icc'
            cxx: 'icpc'
            fc: 'ifort'
            module:
              purge: True
              load:
              - gcc/7.2.0
              - intel/2019
        cray:
          craype@2.6.2:
            cc: 'cc'
            cxx: 'CC'
            fc: 'fc'
            module:
              load: [craype/2.6.2]
              swap: [PrgEnv-gnu, PrgEnv-cray]

        clang:
          clang@12.0.0:
            cc: 'clang'
            cxx: 'clang++'
            fc: 'None'
            module:
              load: [clang/12.0]
        cuda:
          cuda@11.0:
            cc: 'nvcc'
            cxx: 'nvcc'
            fc: 'None'
            module:
              load: [cuda/11.0]
        pgi:
          pgi@18.0:
            cc: 'pgcc'
            cxx: 'pgc++'
            fc: 'pgfortran'
            module:
              load: [pgi/18.0]

File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/settings.schema.json/valid/combined_executor.yml
________________________________________________________________________________
system:
  generic:
    hostnames: ['.*']

    moduletool: N/A
    load_default_buildspecs: True
    executors:
      local:
        bash:
          description: submit jobs on local machine
          shell: bash -v

      slurm:
        haswell:
          launcher: sbatch
          options:
            - "-p haswell"
            - "-t 00:10"

      lsf:
        batch:
          launcher: bsub
          queue: batch
          options:
            - "-q batch"
            - "-t 00:10"
      cobalt:
        normal:
          launcher: qsub
          queue: normal
          options:
            - "-n 1"
            - "-t 10"

    compilers:
      compiler:
        gcc:
          default:
            cc: /usr/bin/gcc
            cxx: /usr/bin/g++
            fc: /usr/bin/gfortran

Global Schema

This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema

Schema Content

$ buildtest schema -n global.schema.json --json
{
  "$id": "global.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "global schema",
  "description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
  "type": "object",
  "required": [
    "version",
    "buildspecs"
  ],
  "additionalProperties": false,
  "properties": {
    "version": {
      "type": "string",
      "description": "The semver version of the schema to use (x.x)."
    },
    "maintainers": {
      "type": "array",
      "description": "One or more maintainers or aliases",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
    "buildspecs": {
      "type": "object",
      "description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
      "propertyNames": {
        "pattern": "^[A-Za-z_.][A-Za-z0-9_.]*$",
        "maxLength": 32
      }
    }
  }
}

Schema Examples

$ buildtest schema -n global.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"

buildspecs:
  # testing all caps
  ABCDEFGHIJKLMNOPQRSTUVWXYZ:
    type: script
    run: "hostname"

  # testing all lowercase letters
  abcdefghijklmnopqrstuvwxyz:
    type: script
    run: "hostname"

  # testing '_' in beginning followed by all numbers
  _0123456789:
    type: script
    run: "hostname"

  # testing '_' in combination with caps, lowercase and numbers
  _ABCDEFabcdef0123456789:
    type: script
    run: "hostname"

  # testing '_' at end of word
  abcdefghijklmnopqrstuvwxyz_:
    type: script
    run: "hostname"

  # testing '.' in beginning of word
  .helloworld:
    type: script
    run: hostname

  # testing '.' in middle of word
  hello.world:
    type: script
    run: hostname

  # testing '.' at end of word
  helloworld.:
    type: script
    run: hostname

File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/invalid/missing-version.yml
________________________________________________________________________________
buildspecs:
  # Shell would be accepted to indicate a single line shell command (or similar)
  login_node_check:
    type: script
    run: "ping login 1"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/invalid/exceed_testname_length.yml
________________________________________________________________________________
# this test fails because it exceeds 32 character length for test name
version: "1.0"
buildspecs:
  this_test_exceeds_32_character_length:
    type: script
    run: hostname
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/invalid/unique_maintainers.yml
________________________________________________________________________________
version: "1.0"
maintainers: [shahzebsiddiqui, shahzebsiddiqui]
buildspecs:
  hostname:
    type: script
    run: "hostname"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/invalid/maintainers_type_mismatch.yml
________________________________________________________________________________
version: "1.0"
# wrong type for maintainers key, expects a string
maintainers: 1
buildspecs:
  hostname:
    type: script
    run: "hostname"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/global.schema.json/invalid/invalid_pattern.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
  # invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$" when declaring a dict
  (badname:
    type: script
    run: "ping login 1"

Script Schema

This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when type: script is specified. For more details on script schema see Script Schema.

Schema Content

$ buildtest schema -n script-v1.0.schema.json --json
{
  "$id": "script-v1.0.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "script schema version 1.0",
  "description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
  "type": "object",
  "required": [
    "type",
    "run",
    "executor"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^script$",
      "description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "batch": {
      "$ref": "definitions.schema.json#/definitions/batch"
    },
    "sbatch": {
      "$ref": "definitions.schema.json#/definitions/sbatch"
    },
    "bsub": {
      "$ref": "definitions.schema.json#/definitions/bsub"
    },
    "cobalt": {
      "$ref": "definitions.schema.json#/definitions/cobalt"
    },
    "pbs": {
      "$ref": "definitions.schema.json#/definitions/pbs"
    },
    "BB": {
      "$ref": "definitions.schema.json#/definitions/BB"
    },
    "DW": {
      "$ref": "definitions.schema.json#/definitions/DW"
    },
    "env": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "vars": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "run_only": {
      "$ref": "definitions.schema.json#/definitions/run_only"
    },
    "shell": {
      "type": "string",
      "description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed",
      "pattern": "^(/bin/bash|/bin/sh|/bin/csh|/bin/tcsh|/bin/zsh|bash|sh|csh|tcsh|zsh|python).*"
    },
    "shebang": {
      "type": "string",
      "description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
    },
    "run": {
      "type": "string",
      "description": "A script to run using the default shell."
    },
    "status": {
      "$ref": "definitions.schema.json#/definitions/status"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    },
    "executors": {
      "$ref": "definitions.schema.json#/definitions/executors"
    }
  }
}

Schema Examples

$ buildtest schema -n script-v1.0.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/script-v1.0.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
  multiline_run:
    executor: generic.local.bash
    type: script
    description: multiline run command
    run: |
      echo "1"
      echo "2"

  single_command_run:
    executor: generic.local.bash
    type: script
    description: single command as a string for run command
    run: "hostname"

  declare_env:
    executor: generic.local.bash
    type: script
    description: declaring environment variables
    env:
      FOO: BAR
      X: 1
    run: |
      echo $FOO
      echo $X

  declare_vars:
    executor: generic.local.bash
    type: script
    description: declaring variables
    vars:
      First: Bob
      Last:  Bill
    run: |
      echo "First:" $First
      echo "Last:" $Last


  declare_shell_sh:
    executor: generic.local.sh
    type: script
    description: declare shell name to sh
    shell: sh
    run: hostname

  declare_shell_bash:
    executor: generic.local.bash
    type: script
    description: declare shell name to bash
    shell: bash
    run: hostname

  declare_shell_python:
    executor: generic.local.python
    type: script
    description: declare shell name to python
    shell: python
    run: |
      print("Hello World")

  declare_shell_bin_bash:
    executor: generic.local.bash
    type: script
    description: declare shell name to /bin/bash
    shell: "/bin/bash -e"
    run: hostname

  declare_shell_name_bin_sh:
    executor: generic.local.sh
    type: script
    description: declare shell name to /bin/sh
    shell: "/bin/sh -e"
    run: hostname

  declare_shell_opts:
    executor: generic.local.sh
    type: script
    description: declare shell name to sh
    shell: "sh -e"
    run: hostname

  declare_shell_bin_zsh:
    executor: generic.local.zsh
    type: script
    description: declare shell zsh
    shell: "zsh"
    run: hostname

  declare_shell_zsh:
    executor: generic.local.zsh
    type: script
    description: declare shell /bin/zsh
    shell: "zsh"
    run: hostname

  declare_shell_bin_csh:
    executor: generic.local.csh
    type: script
    description: declare shell /bin/csh
    shell: "/bin/csh"
    run: hostname

  declare_shell_csh:
    executor: generic.local.csh
    type: script
    description: declare shell /bin/tcsh
    shell: "csh"
    run: hostname

  declare_shell_bin_tcsh:
    executor: generic.local.csh
    type: script
    description: declare shell /bin/tcsh
    shell: "/bin/tcsh"
    run: hostname


  declare_shell_tcsh:
    executor: generic.local.csh
    type: script
    description: declare shell tcsh
    shell: "tcsh"
    run: hostname

  declare_shebang:
    executor: generic.local.bash
    type: script
    description: declare shell name to sh
    shebang: "#!/usr/bin/env bash"
    run: hostname

  status_returncode_list:
    executor: generic.local.bash
    type: script
    description: The returncode can be a list of integers
    run: exit 0
    status:
      returncode: [0]

  status_returncode_int:
    executor: generic.local.bash
    type: script
    description: The returncode can be an integer to match with single returncode
    run: exit 0
    status:
      returncode: 0

  status_regex:
    executor: generic.local.bash
    type: script
    description: This test pass with a regular expression status check
    run: hostname
    status:
      regex:
        stream: stdout
        exp: "^$"

  status_regex_returncode:
    executor: generic.local.bash
    type: script
    description: This test fails because returncode and regex specified
    run: hostname
    status:
      returncode: [0]
      regex:
        stream: stdout
        exp: "^hello"


  status_runtime_min_max:
    type: script
    executor: generic.local.sh
    description: "Run a sleep job for 2 seconds and test pass if its within 1.0-4.0sec"
    tags: ["tutorials"]
    run: sleep 2
    status:
      runtime:
        min: 1.0
        max: 4.0

  status_runtime_min:
    type: script
    executor: generic.local.sh
    description: "Run a sleep job for 2 seconds and test pass if exceeds mintime of 1.0sec"
    tags: ["tutorials"]
    run: sleep 2
    status:
      runtime:
        min: 1.0

  status_runtime_max:
    type: script
    executor: generic.local.sh
    description: "Run a sleep job for 2 seconds and test pass if less than maxtime of 4.0sec"
    tags: ["tutorials"]
    run: sleep 2
    status:
      runtime:
        max: 4.0

  sbatch_example:
    type: script
    executor: generic.local.bash
    description: This test runs hostname using sbatch directives
    sbatch:
      - "-t 10:00:00"
      - "-p normal"
      - "-N 1"
      - "-n 8"
    run: hostname

  bsub_example:
    type: script
    executor: generic.local.bash
    description: This test runs hostname using bsub directives
    bsub:
      - "-W 00:30"
      - "-N 1"
    run: hostname

  cobalt_example:
    type: script
    executor: generic.local.bash
    description: This test runs hostname using cobalt directives
    cobalt:
      - "-t 30"
      - "-n 1"
    run: hostname

  skip_example:
    type: script
    executor: generic.local.bash
    description: this test is skip
    skip: true
    run: hostname

  tag_str_example:
    type: script
    executor: generic.local.bash
    description: tags can be defined as string
    tags: network
    run: hostname

  tag_list_example:
    type: script
    executor: generic.local.bash
    description: This is a tag example using list
    sbatch:
      - "-t 10:00:00"
      - "-p normal"
      - "-N 1"
      - "-n 8"
    tags: ["slurm"]
    run: hostname

  run_only_example:
    type: script
    executor: generic.local.bash
    description: run_only example that runs with user1 on Linux system (rhel, centos) with LSF
    run_only:
      user: user1
      scheduler: lsf
      platform: Linux
      linux_distro:
        - rhel
        - centos
    run: |
      uname -av
      lsinfo

  metrics_regex_example:
    type: script
    executor: generic.local.bash
    description: metrics regular expression example
    run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
    metrics:
      hpcg_rating:
        regex:
          exp: 'rating of=(\d+\.\d+)$'
          stream: stdout

  metric_variable_assignment:
    executor: generic.local.sh
    type: script
    description: capture result metric based on variables and environment variable
    vars:
      GFLOPS: "63.6515"
    env:
      FOO: BAR
    run: |
      echo $GFLOPS
      echo $FOO
    tags: tutorials
    metrics:
      gflops:
        vars: "GFLOPS"
      foo:
        env: "FOO"

  multi_executor_vars:
    type: script
    executor: 'generic.local.(sh|bash)'
    description: single test multiple executor with variable declaration
    run: |
      echo $X
      echo $Y
    executors:
      generic.local.sh:
        vars:
          X: 1
          Y: 2
      generic.local.bash:
        vars:
          X: 10
          Y: 11

  multi_executor_environment:
    type: script
    executor: 'generic.local.(sh|bash)'
    description: single test multiple executor with environment declaration
    run: echo $SHELL
    executors:
      generic.local.sh:
        env:
          SHELL: sh
      generic.local.bash:
        env:
          SHELL: bash

  executors_sbatch_declaration:
    type: script
    executor: 'generic.local.(bash|sh)'
    description: Declaring sbatch by executors section
    tags: [tutorials]
    run: hostname
    sbatch: ["-N 4"]
    executors:
      generic.local.bash:
        sbatch: ["-n 4", "-N 1", "-t 30"]
      generic.local.sh:
        sbatch: ["-n 8", "-N 1", "-t 60"]

  executors_bsub_declaration:
    type: script
    executor: 'generic.local.(bash|sh)'
    description: Declaring bsub  by executors section
    tags: [tutorials]
    run: hostname
    executors:
      generic.local.bash:
        bsub: ["-n 4", "-W 30"]
      generic.local.sh:
        bsub: ["-n 8", "-W 60"]

  executors_pbs_declaration:
    type: script
    executor: 'generic.local.(bash|sh)'
    description: Declaring pbs by executors section
    tags: [tutorials]
    run: hostname
    executors:
      generic.local.bash:
        pbs: ["-l ncpus=4", "-l walltime=30"]
      generic.local.sh:
        pbs: ["-l ncpus=8", "-l walltime=60"]

  executors_status_declaration:
    type: script
    executor: 'generic.local.(bash|sh)'
    description: Declaring status by executors section
    tags: [tutorials]
    run: exit 0
    executors:
      generic.local.bash:
        status:
          returncode: 0
      generic.local.sh:
        status:
          returncode: [1, 2]

  executors_metrics_declaration:
    type: script
    executor: 'generic.local.(bash|sh)'
    description: Declaring metrics by executors section
    tags: [tutorials]
    run: echo "Hello World"
    executors:
      generic.local.bash:
        metrics:
          hello:
            regex:
              stream: stdout
              exp: "(Hello)"
      generic.local.sh:
        metrics:
          world:
            regex:
              stream: stdout
              exp: "(World)"
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/script-v1.0.schema.json/invalid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
  invalid_test_name_&!@#$%:
    type: script
    executor: generic.local.bash
    description: "invalid test name"

  invalid_bash:
    type: script
    executor: generic.local.bash
    shell: "bash-missing-run"

  missing_run_key:
    type: script
    executor: generic.local.bash
    description: invalid key name roon, missing run key
    roon: |
        systemctl is-active slurmd
        systemctl is-enabled slurmd | grep enabled

  invalid_env_type:
    type: script
    executor: generic.local.bash
    description: env key should be a dictionary
    env:
      - FOO=BAR
    run: echo $FOO

  invalid_vars_type:
    type: script
    executor: generic.local.bash
    description: var key should be a dictionary
    vars:
      - FOO=BAR
    run: echo $FOO


  invalid_description:
    type: script
    executor: generic.local.bash
    description:
      - "Multi Line description"
      - "is not accepted"

  invalid_regex_stream:
    type: script
    executor: generic.local.bash
    description: This test fails because of invalid regex stream
    run: hostname
    status:
      regex:
        stream: file
        exp: "world$"


  regex_additionalProperties_test:
    type: script
    executor: generic.local.bash
    description: Testing for additional properties in regex field
    run: hostname
    status:
      regex:
        stream: stdout
        exp: "world$"
        X: 1

  missing_regex_exp:
    type: script
    executor: generic.local.bash
    description: This test fails because of missing key 'exp' in regex
    run: hostname
    status:
      regex:
        stream: stdout

  invalid_returncode_type:
    type: script
    executor: generic.local.bash
    description: This test fails because of invalid return code type
    run: hostname
    status:
      returncode: ["1"]

  empty_returncode_list:
    type: script
    executor: generic.local.bash
    description: An empty returncode list will cause an error
    run: hostname
    status:
      returncode: []

  non_int_returncodes:
    type: script
    executor: generic.local.bash
    description: The returncode must be an int and not a number
    run: exit 1
    status:
      returncode:  1.01

  non_int_returncodes_list:
    type: script
    executor: generic.local.bash
    description: The returncode must be a list of integers and no numbers
    run: exit 1
    status:
      returncode:  [1, 2.230]

  invalid_shell_usr_bin_bash:
    type: script
    executor: generic.local.bash
    description: invalid shell name '/usr/bin/bash'
    shell: /usr/bin/bash
    run: hostname

  invalid_shell_type:
    type: script
    executor: generic.local.bash
    description: invalid shell type must be a string
    shell: ["/bin/bash"]
    run: hostname

  invalid_type_shell_shebang:
    type: script
    executor: generic.local.bash
    description: invalid type for shell shebang, must be a string
    shebang: ["#!/bin/bash"]
    run: hostname

  invalid_skip_value:
    type: script
    executor: generic.local.bash
    description: invalid value for skip, must be boolean
    skip: 1
    run: hostname

  empty_tags:
    type: script
    executor: generic.local.bash
    description: tag list can't be empty, requires one item.
    tags: []
    run: hostname

  non_unique_tags:
    type: script
    executor: local.bash
    description: tag names must be unique
    tags: ["network", "network"]
    run: hostname

  invalid_tags_value:
    type: script
    executor: generic.local.bash
    description: invalid tag value must be all string items
    tags: ["network", 400 ]
    run: hostname

  additionalProperties_test:
    type: script
    executor: generic.local.bash
    description: additional properties are not allowed so any invalid key/value pair will result in error
    FOO: BAR
    run: hostname

  additionalProperties_status:
    type: script
    executor: generic.local.bash
    description: test additional properties in status object. This is not allowed
    sbatch: [ "-n 2", "-q normal", "-t 10"]
    run: hostname
    status:
      slurm_job_state: "COMPLETED"
      FOO: BAR

  invalid_runtime_min:
    type: script
    executor: generic.local.sh
    description: "Invalid type for min property in runtime. Must be an integer or float not a string"
    run: sleep 2
    status:
      runtime:
        min: "1"

  runtime_min_must_exceed_0:
    type: script
    executor: generic.local.sh
    description: "The runtime must exceed 0"
    run: sleep 2
    status:
      runtime:
        min: -1

  invalid_slurm_job_state:
    type: script
    executor: generic.local.sh
    description: invalid value for slurm_job_state, should raise error with enum values.
    sbatch:
      - "-n 2"
      - "-q normal"
      - "-t 10"
    run: hostname
    status:
      slurm_job_state: "FINISH"

  duplicate_linux_distro:
    type: script
    executor: generic.local.bash
    description: Duplicate items in linux_distro is not allowed
    run_only:
      linux_distro:
        - rhel
        - rhel
    run: uname -av

  empty_list_linux_distro:
    type: script
    executor: generic.local.bash
    description: Empty List in linux_distro is not allowed. Requires atleast 1 item
    run_only:
      linux_distro: []
    run: uname -av

  additionalProperties_run_only:
    type: script
    executor: generic.local.bash
    description: additional Properties not allowed in run_only field. Invalid field python
    run_only:
      user: root
      python: 3.5
    run: hostname

  invalid_metrics_additional_property:
    type: script
    executor: generic.local.bash
    description: Test for additional property for metrics property
    vars:
      FOO: BAR
    run: echo $FOO
    metrics:
      foo:
        variable: FOO

  invalid_metrics_type:
    type: script
    executor: generic.local.bash
    description: metrics property is an object, testing for type
    vars:
      FOO: BAR
    run: echo $FOO
    metrics: FOO

  executors_invalid_var_type:
    type: script
    executor: "generic.local.(bash|sh|zsh)"
    description: Invalid type field for 'vars'
    tags: [tutorials]
    run: echo $FOO
    executors:
      generic.local.bash:
        vars: ["FOO=BAR"]

  executors_additionalProperties:
    type: script
    executor: "generic.local.(bash|sh|zsh)"
    description: Testing for additional properties in 'executors'
    tags: [tutorials]
    run: hostname
    sbatch: ["-N 4"]
    executors:
      generic.local.bash:
        sbatch: ["-n 4", "-N 1", "-t 30"]
        FOO: BAR
      generic.local.sh:
        sbatch: ["-n 8", "-N 1", "-t 60"]
      generic.local.zsh:
        sbatch: ["-n 16", "-N 2", "-t 120"]

Compiler Schema

This is the compiler schema used for validating buildspecs that define test using type: compiler. This schema is used for compiling a single source code. For more details see Compiler Schema

Schema Content

$ buildtest schema -n compiler-v1.0.schema.json --json
{
  "$id": "compiler-v1.0.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "compiler schema version 1.0",
  "description": "The compiler schema is of ``type: compiler`` in sub-schema which is used for compiling and running programs",
  "type": "object",
  "required": [
    "type",
    "source",
    "compilers",
    "executor"
  ],
  "definitions": {
    "cc": {
      "type": "string",
      "description": "Set C compiler wrapper"
    },
    "fc": {
      "type": "string",
      "description": "Set Fortran compiler wrapper"
    },
    "cxx": {
      "type": "string",
      "description": "Set C++ compiler wrapper"
    },
    "cflags": {
      "type": "string",
      "description": "Set C compiler flags."
    },
    "fflags": {
      "type": "string",
      "description": "Set Fortran compiler flags."
    },
    "cxxflags": {
      "type": "string",
      "description": "Set C++ compiler flags."
    },
    "ldflags": {
      "type": "string",
      "description": "Set linker flags"
    },
    "cppflags": {
      "type": "string",
      "description": "Set C or C++ preprocessor flags"
    },
    "pre_build": {
      "type": "string",
      "description": "Run commands before building program"
    },
    "post_build": {
      "type": "string",
      "description": "Run commands after building program"
    },
    "pre_run": {
      "type": "string",
      "description": "Run commands before running program"
    },
    "post_run": {
      "type": "string",
      "description": "Run commands after running program"
    },
    "run": {
      "type": "string",
      "description": "Run command for launching compiled binary"
    },
    "default_compiler_all": {
      "type": "object",
      "description": "Specify compiler configuration for all compiler groups. Use the ``all`` property if configuration is shared across compiler groups. This property can be overridden in compiler group or named compiler in ``config`` section.",
      "additionalProperties": false,
      "properties": {
        "sbatch": {
          "$ref": "definitions.schema.json#/definitions/sbatch"
        },
        "bsub": {
          "$ref": "definitions.schema.json#/definitions/bsub"
        },
        "cobalt": {
          "$ref": "definitions.schema.json#/definitions/cobalt"
        },
        "pbs": {
          "$ref": "definitions.schema.json#/definitions/pbs"
        },
        "batch": {
          "$ref": "definitions.schema.json#/definitions/batch"
        },
        "BB": {
          "$ref": "definitions.schema.json#/definitions/BB"
        },
        "DW": {
          "$ref": "definitions.schema.json#/definitions/DW"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "pre_build": {
          "$ref": "#definitions/pre_build"
        },
        "post_build": {
          "$ref": "#definitions/post_build"
        },
        "pre_run": {
          "$ref": "#definitions/pre_run"
        },
        "post_run": {
          "$ref": "#definitions/post_run"
        },
        "run": {
          "$ref": "#definitions/run"
        }
      }
    },
    "default_compiler_config": {
      "type": "object",
      "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
      "properties": {
        "cc": {
          "$ref": "#definitions/cc"
        },
        "fc": {
          "$ref": "#definitions/fc"
        },
        "cxx": {
          "$ref": "#definitions/cxx"
        },
        "cflags": {
          "$ref": "#definitions/cflags"
        },
        "fflags": {
          "$ref": "#definitions/fflags"
        },
        "cxxflags": {
          "$ref": "#definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "#definitions/ldflags"
        },
        "cppflags": {
          "$ref": "#definitions/cppflags"
        },
        "sbatch": {
          "$ref": "definitions.schema.json#/definitions/sbatch"
        },
        "bsub": {
          "$ref": "definitions.schema.json#/definitions/bsub"
        },
        "cobalt": {
          "$ref": "definitions.schema.json#/definitions/cobalt"
        },
        "pbs": {
          "$ref": "definitions.schema.json#/definitions/pbs"
        },
        "batch": {
          "$ref": "definitions.schema.json#/definitions/batch"
        },
        "BB": {
          "$ref": "definitions.schema.json#/definitions/BB"
        },
        "DW": {
          "$ref": "definitions.schema.json#/definitions/DW"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "pre_build": {
          "$ref": "#definitions/pre_build"
        },
        "post_build": {
          "$ref": "#definitions/post_build"
        },
        "pre_run": {
          "$ref": "#definitions/pre_run"
        },
        "post_run": {
          "$ref": "#definitions/post_run"
        },
        "run": {
          "$ref": "#definitions/run"
        }
      }
    },
    "compiler_declaration": {
      "type": "object",
      "description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
      "additionalProperties": false,
      "properties": {
        "cc": {
          "$ref": "#definitions/cc"
        },
        "fc": {
          "$ref": "#definitions/fc"
        },
        "cxx": {
          "$ref": "#definitions/cxx"
        },
        "cflags": {
          "$ref": "#definitions/cflags"
        },
        "fflags": {
          "$ref": "#definitions/fflags"
        },
        "cxxflags": {
          "$ref": "#definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "#definitions/ldflags"
        },
        "cppflags": {
          "$ref": "#definitions/cppflags"
        },
        "sbatch": {
          "$ref": "definitions.schema.json#/definitions/sbatch"
        },
        "bsub": {
          "$ref": "definitions.schema.json#/definitions/bsub"
        },
        "cobalt": {
          "$ref": "definitions.schema.json#/definitions/cobalt"
        },
        "pbs": {
          "$ref": "definitions.schema.json#/definitions/pbs"
        },
        "batch": {
          "$ref": "definitions.schema.json#/definitions/batch"
        },
        "BB": {
          "$ref": "definitions.schema.json#/definitions/BB"
        },
        "DW": {
          "$ref": "definitions.schema.json#/definitions/DW"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "pre_build": {
          "$ref": "#definitions/pre_build"
        },
        "post_build": {
          "$ref": "#definitions/post_build"
        },
        "pre_run": {
          "$ref": "#definitions/pre_run"
        },
        "post_run": {
          "$ref": "#definitions/post_run"
        },
        "run": {
          "$ref": "#definitions/run"
        },
        "module": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "purge": {
              "type": "boolean",
              "description": "Run ``module purge`` if purge is set"
            },
            "load": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "Load one or more modules via ``module load``"
            },
            "restore": {
              "description": "Load a collection name via ``module restore``",
              "type": "string"
            },
            "swap": {
              "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
              "type": "array",
              "uniqueItems": true,
              "minItems": 2,
              "maxItems": 2,
              "items": {
                "type": "string"
              }
            }
          }
        }
      }
    }
  },
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^compiler$",
      "description": "Select schema type to use when validating buildspec. This must be of set to ``compiler``."
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "compilers": {
      "type": "object",
      "required": [
        "name"
      ],
      "additionalProperties": false,
      "properties": {
        "name": {
          "description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "exclude": {
          "description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "default": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "all": {
              "$ref": "#definitions/default_compiler_all"
            },
            "gcc": {
              "$ref": "#definitions/default_compiler_config"
            },
            "intel": {
              "$ref": "#definitions/default_compiler_config"
            },
            "pgi": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cray": {
              "$ref": "#definitions/default_compiler_config"
            },
            "clang": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cuda": {
              "$ref": "#definitions/default_compiler_config"
            },
            "upcxx": {
              "$ref": "#definitions/default_compiler_config"
            }
          }
        },
        "config": {
          "type": "object",
          "description": "Specify compiler configuration based on named compilers.",
          "patternProperties": {
            "^.*$": {
              "$ref": "#definitions/compiler_declaration"
            }
          }
        }
      }
    },
    "source": {
      "type": "string",
      "description": "Specify a source file for compilation, the file can be relative path to buildspec or an absolute path"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "run_only": {
      "$ref": "definitions.schema.json#/definitions/run_only"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    }
  }
}

Schema Examples

$ buildtest schema -n compiler-v1.0.schema.json --example
File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/compiler-v1.0.schema.json/valid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
  gnu_example:
    executor: local.bash
    type: compiler
    description: "gnu example with modules, and cflags example"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc@8.4.0:
          cflags: "-O3"

  intel_example:
    executor: local.bash
    type: compiler
    description: "intel example using cflags"
    source: src/hello.c
    compilers:
      name: [intel]
      config:
        intel@2018:
          cflags: "-O1"

  clang_example:
    executor: local.bash
    type: compiler
    description: "clang example using cflags"
    source: src/hello.c
    compilers:
      name: [clang]
      default:
        clang:
          cflags: "-O1"
      config:
        clang@11:
          cflags: "-O2"

  upcxx_example:
    executor: local.bash
    type: compiler
    description: "upcxx compiler declaration in default and config section "
    source: src/hello.c
    compilers:
      name: [upcxx]
      default:
        upcxx:
          cflags: "-g aries"
      config:
        upcxx@2020:
          cflags: "-O1 -g aries"

  pgi_example:
    executor: local.bash
    type: compiler
    description: "pgi example using cxxflags, ldflags in default and config section"
    source: src/hello.cpp
    compilers:
      name: ["^(pgi|PrgEnv)"]
      default:
        pgi:
          cxxflags: "-O1"
          ldflags: "-lm"
      config:
        pgi@18.1:
          module:
            swap: [PrgEnv-gnu, PrgEnv-pgi]
            load: [pgi/18.1]
        pgi@18.2:
          module:
            swap: [PrgEnv-gnu, PrgEnv-pgi]
            load: [pgi/18.2]

  cray_example:
    executor: local.bash
    type: compiler
    description: "cray example using fflags and cppflags"
    source: src/hello.f90
    compilers:
      name: ["PrgEnv-cray"]
      default:
        cray:
          fflags: "-O1"
      config:
        PrgEnv-cray@2.6.2:
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]


  sbatch_example_all_compiler_groups:
    type: compiler
    description: sbatch example to for all compiler groups
    executor: local.bash
    source: src/hello.f90
    compilers:
      name: ["PrgEnv-cray"]
      default:
        cray:
          fflags: "-O1"
        all:
          sbatch: ["-t 10", "-n 2", "-C haswell" ]
      config:
        PrgEnv-cray@2.6.2:
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

  bsub_all_compiler_groups:
    type: compiler
    description: bsub example for all compiler groups
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [intel]
      default:
        all:
          bsub: ["-W 00:30", "-n 2"]
      config:
        intel@2019:
          cxxflags: "-O1"

  cobalt_all_compiler_groups:
    type: compiler
    description: cobalt example for all compiler groups
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [intel]
      default:
        all:
          cobalt: ["-t 30", "-n 1"]
      config:
        intel@2019:
          cxxflags: "-O1"

  sbatch_compiler_group:
    type: compiler
    description: sbatch example in multiple compiler groups.
    executor: local.bash
    source: src/hello.f90
    compilers:
      name: ["^(gcc|intel)"]
      default:
        gcc:
          fflags: "-O1"
          sbatch: ["-t 10", "-n 2", "-C haswell" ]
        intel:
          fflags: "-O2"
          sbatch: ["-t 10", "-n 2", "-C knl" ]
      config:
        gcc@8.1.0:
          sbatch: ["-t 60", "-n 2", "-C knl"]
          module:
            swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]


  bsub_compiler_group:
    type: compiler
    description: bsub example in multiple compiler groups.
    executor: local.bash
    source: src/hello.f90
    compilers:
      name: ["^(gcc|intel)"]
      default:
        gcc:
          fflags: "-O1"
          bsub: ["-W 00:30", "-n 2" ]
        intel:
          fflags: "-O2"
          bsub: ["-W 00:30", "-n 4" ]
      config:
        gcc@8.1.0:
           bsub: ["-W 00:30", "-n 6" ]
           module:
             swap: [PrgEnv-intel, PrgEnv-gnu/6.1.0]

  batch_example:
    type: compiler
    description: example using batch field
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [intel]
      default:
        all:
          batch:
            "timelimit": "30"
            "nodecount": "2"
            "queue": "batch"
            "account": "biology"
      config:
        intel@2019:
          cxxflags: "-O1"

  env_example:
    type: compiler
    description: Setting environment variables
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: ["^(gcc)"]
      default:
        all:
          env:
            OMP_NUM_THREADS: 2
          run: $_EXEC 1 2 4
      config:
        gcc@10.2.0:
          cxxflags: "-fopenmp"

  custom_env_by_compiler_group:
    type: compiler
    description: Setting environment variables in compiler groups
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: ["^(gcc|pgi)"]
      default:
        all:
          run: $_EXEC 1 2 4
        gcc:
          cxxflags: "-fopenmp"
          env:
            OMP_NUM_THREADS: 4
        pgi:
          cxxflags: "-mp"
          env:
            OMP_NUM_THREADS: 6
      config:
        gcc@10.2.0:
          env:
            OMP_NUM_THREADS: 6

        gcc@9.2.0:
          env:
            OMP_NUM_THREADS: 8

        pgi@9.2.0:
          env:
            OMP_NUM_THREADS: 10

  vars_example:
    type: compiler
    description: Setting shell variables
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: ["^(gcc)"]
      default:
        all:
          vars:
            OUTFILE: /tmp/file1.txt
          run: $_EXEC > $OUTFILE
      config:
        gcc@10.2.0:
          cxxflags: "-fopenmp"

  pass_args_run:
    type: compiler
    description: Passing arguments to executable in run section
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [intel]
      default:
        all:
          run: $_EXEC 1 2 4
      config:
        intel@2019:
          cxxflags: "-O1"

  mpi_launcher_example:
    type: compiler
    description: mpi launcher example
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [gcc]
      default:
        all:
          run: mpirun -np 2 $_EXEC
      config:
        gcc@7.3.0:
          cflags: "-O3"
          cxx: mpicxx

  status_returncode_example:
    type: compiler
    description: Status returncode match example
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [gnu]
      default:
        all:
          vars:
            OUTFILE: /tmp/file1.txt
          run: $_EXEC > $OUTFILE
          status:
            returncode: 1
      config:
        gcc@10.2.0:
          cxxflags: "-fopenmp"

  pre_post_build_run_sections:
    type: compiler
    description: Run commands pre and post build section
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: ["^(gcc)"]
      default:
        all:
          pre_build: echo "pre-build section for ALL compilers"
          post_build: echo "post-build section for ALL Compilers"
          pre_run: echo "pre-run section for ALL compilers"
          post_run: echo "post-run section for ALL Compilers"
        gcc:
          pre_build: echo "pre-build section for GCC compilers"
          post_build: echo "post-build section for GCC compilers"
          pre_run: echo "pre-run section for ALL compilers"
          post_run: echo "post-run section for ALL Compilers"
      config:
        gcc@7.3.0:
          pre_build: echo "pre-build section for gcc@7.3.0"
          post_build: echo "post-build section for gcc@7.3.0"
          pre_run: echo "pre-run section for ALL compilers"
          post_run: echo "post-run section for ALL Compilers"
          cflags: "-O3"
        gcc@8.2.0:
          pre_build: echo "gcc --version"
          cflags: "-O3"


  multi_compilers:
    type: compiler
    description: Select one or more compilers to run test
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: ["^(gcc|intel|pgi|cray)"]
      exclude: [intel@18]
      default:
        gcc:
          cflags: "-fopenmp"
        intel:
          cflags: "-qopenmp"
        pgi:
          cflags: "-fopenmp"
        cray:
          cflags: "-h omp"
      config:
        gcc@7.5.0:
          cflags: "-O3"
          module:
            load: [gcc/7.5.0]
        intel@17:
          module:
            load: [intel/2017]
        intel@18:
          module:
            load: [intel/2018]
        pgi/18.0:
          module:
            load: [pgi/18.0]
        craype/2.6.2:
          module:
            swap: [PrgEnv-intel, PrgEnv-cray]
            load: [craype/2.6.2]

  metrics_example:
    type: compiler
    description: Recording test metrics with compiler schema
    executor: local.bash
    source: "src/hello.cpp"
    compilers:
      name: [gnu]
      default:
        all:
          vars:
            OUTFILE: /tmp/file1.txt
          run: $_EXEC > $OUTFILE
          status:
            returncode: 1
      config:
        gcc@10.2.0:
          cxxflags: "-fopenmp"
    metrics:
      outfile:
        vars: "OUTFILE"

File: /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/v0.10.2/buildtest/schemas/examples/compiler-v1.0.schema.json/invalid/examples.yml
________________________________________________________________________________
version: "1.0"
buildspecs:
  missing_type:
    executor: local.bash
    description: "type key is missing, this is a required field"
    source: "src/hello.c"
    compilers:
      name: [intel]

  missing_required_compilers:
    executor: local.bash
    type: compiler
    description: "missing required field compilers "
    source: "src/hello.c"

  missing_required_source:
    executor: local.bash
    type: compiler
    description: "missing required field 'source' "
    compilers:
      name: [gcc]

  invalid_type_value:
    executor: local.bash
    type: script
    description: "invalid value for type field must be 'compiler' "
    source: src/hello.c
    compilers:
      name: [gcc]

  invalid_description_value:
    executor: local.bash
    type: compiler
    description: 1
    source: src/hello.c
    compilers:
      name: [gcc]

  invalid_type_module:
    executor: local.bash
    type: compiler
    description: "type for 'module' key, expecting a property but received 'string' "
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module: "module load gcc/9.2.0"

  module_purge_invalid_type:
    executor: local.bash
    type: compiler
    description: "The purge property module is invalid. Expects bool got an int"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            purge: 1

  module_swap_duplicate_check:
    executor: local.bash
    type: compiler
    description: "The swap property expects two unique items"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            swap: [gcc/8.0, gcc/8.0]

  module_swap_min_items:
    executor: local.bash
    type: compiler
    description: "The swap property expects a minimum of 2 items"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            swap: [gcc/8.0]

  module_swap_max_items:
    executor: local.bash
    type: compiler
    description: "The swap property expects a maximum of 2 items"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            swap: [gcc/8.0, gcc/9.0, gcc/10.0]

  module_load_duplicate_items:
    executor: local.bash
    type: compiler
    description: "The load property expects unique items"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            load: [gcc/9.2.0, gcc/9.2.0]

  module_load_min_items:
    executor: local.bash
    type: compiler
    description: "The load property expects a minimum of 1 item"
    source: src/hello.c
    compilers:
      name: [gcc]
      config:
        gcc/9.2.0:
          module:
            load: []

  additionalProperties_main:
    executor: local.bash
    type: compiler
    description: "test additionalProperties in main schema"
    foo: bar
    source: src/hello.c
    compilers:
      name: [gcc]

  missing_required_compiler_name:
    executor: local.bash
    type: compiler
    description: "'name' field in compilers section is required field"
    source: src/hello.f90
    compilers:
    default:
      cray:
        fflags: "-O1"
    config:
      PrgEnv-cray@2.6.2:
        module:
          swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

  uniqueItems_compiler_name:
    executor: local.bash
    type: compiler
    description: "Test unique items in 'name' field in compilers section"
    source: src/hello.f90
    compilers:
      name: ["^(PrgEnv-cray)", "^(PrgEnv-cray)"]
      config:
        PrgEnv-cray@2.6.2:
          fflags: "-O1"
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

  additionalProperties_compiler:
    executor: local.bash
    type: compiler
    description: "Test additionalProperies in compiler section"
    source: src/hello.f90
    compilers:
      name: ["PrgEnv-cray"]
      FOO: BAR
      default:
        all:
          env:
            X: 1
      config:
        PrgEnv-cray@2.6.2:
          fflags: "-O1"
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

  additionalProperties_compiler_default_all:
    executor: local.bash
    type: compiler
    description: "Test additionalProperies in compiler default all section"
    source: src/hello.f90
    compilers:
      name: ["PrgEnv-cray"]
      default:
        all:
          XYZ: 123
      config:
        PrgEnv-cray@2.6.2:
          fflags: "-O1"
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

  additionalProperties_compiler_config:
    executor: local.bash
    type: compiler
    description: "Test additionalProperies in compiler config section, FOO: BAR"
    source: src/hello.f90
    compilers:
      name: ["PrgEnv-cray"]
      config:
        PrgEnv-cray@2.6.2:
          FOO: BAR
          fflags: "-O1"
          module:
            swap: [PrgEnv-intel, PrgEnv-cray/2.6.2]

Spack Schema

This schema is used for writing tests with spack package manager using type: spack field. For more details see Spack Schema.

Schema Content

$ buildtest schema -n spack-v1.0.schema.json --json
{
  "$id": "spack-v1.0.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "spack schema version 1.0",
  "description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
  "type": "object",
  "required": [
    "type",
    "executor",
    "spack"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^spack$",
      "description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "env": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "vars": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "sbatch": {
      "$ref": "definitions.schema.json#/definitions/sbatch"
    },
    "bsub": {
      "$ref": "definitions.schema.json#/definitions/bsub"
    },
    "cobalt": {
      "$ref": "definitions.schema.json#/definitions/cobalt"
    },
    "pbs": {
      "$ref": "definitions.schema.json#/definitions/pbs"
    },
    "batch": {
      "$ref": "definitions.schema.json#/definitions/batch"
    },
    "BB": {
      "$ref": "definitions.schema.json#/definitions/BB"
    },
    "DW": {
      "$ref": "definitions.schema.json#/definitions/DW"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "status": {
      "$ref": "definitions.schema.json#/definitions/status"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    },
    "executors": {
      "$ref": "definitions.schema.json#/definitions/executors"
    },
    "pre_cmds": {
      "type": "string",
      "description": "Shell commands run before spack"
    },
    "post_cmds": {
      "type": "string",
      "description": "Shell commands run after spack"
    },
    "spack": {
      "type": "object",
      "description": "Entry point to spack configuration",
      "required": [
        "root"
      ],
      "additionalProperties": false,
      "properties": {
        "root": {
          "type": "string"
        },
        "compiler_find": {
          "type": "boolean",
          "description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror by running ``spack mirror add``"
        },
        "env": {
          "$ref": "#definitions/env",
          "description": "Manage spack environments via ``spack env`` command"
        },
        "install": {
          "$ref": "#definitions/install",
          "description": "Install spack packages by running ``spack install``. "
        },
        "verify_spack": {
          "type": "boolean",
          "description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
          "default": true
        },
        "test": {
          "$ref": "#definitions/test",
          "description": "Entry point to ``spack test``"
        }
      }
    }
  },
  "definitions": {
    "env": {
      "additionalProperties": false,
      "type": "object",
      "description": "Used for managing spack environment using ``spack env`` command. ",
      "properties": {
        "create": {
          "additionalProperties": false,
          "description": "Create a spack environment via ``spack env create``",
          "type": "object",
          "properties": {
            "remove_environment": {
              "type": "boolean",
              "description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
              "default": false
            },
            "name": {
              "type": "string",
              "description": "Name of spack environment to create"
            },
            "manifest": {
              "type": "string",
              "description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env create`` command"
            },
            "dir": {
              "type": "string",
              "description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
            }
          }
        },
        "activate": {
          "additionalProperties": false,
          "type": "object",
          "description": "Activate a spack environment via ``spack env activate``",
          "properties": {
            "name": {
              "type": "string",
              "description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env activate`` command"
            },
            "dir": {
              "type": "string",
              "description": "Activate spack environment from directory."
            }
          }
        },
        "rm": {
          "additionalProperties": false,
          "description": "Remove an existing spack environment via ``spack env rm``.",
          "type": "object",
          "required": [
            "name"
          ],
          "properties": {
            "name": {
              "type": "string",
              "description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
            }
          }
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror in spack environment by running ``spack mirror add``"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
        },
        "concretize": {
          "type": "boolean",
          "description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
        }
      }
    },
    "install": {
      "description": "Install spack packages using ``spack install`` command",
      "additionalProperties": false,
      "type": "object",
      "properties": {
        "options": {
          "type": "string",
          "description": "Pass options to ``spack install`` command"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "List of specs to install using ``spack install`` command"
        }
      }
    },
    "test": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "run"
      ],
      "properties": {
        "remove_tests": {
          "type": "boolean",
          "description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
        },
        "run": {
          "description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
          "type": "object",
          "required": [
            "specs"
          ],
          "additionalProperties": false,
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test run``"
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "List of specs to run tests by running ``spack test run <specs>``."
            }
          }
        },
        "results": {
          "type": "object",
          "description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
          "additionalProperties": false,
          "anyOf": [
            {
              "required": [
                "specs"
              ]
            },
            {
              "required": [
                "suite"
              ]
            },
            {
              "required": [
                "specs",
                "suite"
              ]
            }
          ],
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test results``"
            },
            "suite": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "Report results by  suite name by running ``spack test results <suite>``."
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "Report result by spec name by running ``spack test run -- <specs>``."
            }
          }
        }
      }
    }
  }
}

Schema Examples

$ buildtest schema -n spack-v1.0.schema.json --json
{
  "$id": "spack-v1.0.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "spack schema version 1.0",
  "description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
  "type": "object",
  "required": [
    "type",
    "executor",
    "spack"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^spack$",
      "description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "env": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "vars": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "sbatch": {
      "$ref": "definitions.schema.json#/definitions/sbatch"
    },
    "bsub": {
      "$ref": "definitions.schema.json#/definitions/bsub"
    },
    "cobalt": {
      "$ref": "definitions.schema.json#/definitions/cobalt"
    },
    "pbs": {
      "$ref": "definitions.schema.json#/definitions/pbs"
    },
    "batch": {
      "$ref": "definitions.schema.json#/definitions/batch"
    },
    "BB": {
      "$ref": "definitions.schema.json#/definitions/BB"
    },
    "DW": {
      "$ref": "definitions.schema.json#/definitions/DW"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "status": {
      "$ref": "definitions.schema.json#/definitions/status"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    },
    "executors": {
      "$ref": "definitions.schema.json#/definitions/executors"
    },
    "pre_cmds": {
      "type": "string",
      "description": "Shell commands run before spack"
    },
    "post_cmds": {
      "type": "string",
      "description": "Shell commands run after spack"
    },
    "spack": {
      "type": "object",
      "description": "Entry point to spack configuration",
      "required": [
        "root"
      ],
      "additionalProperties": false,
      "properties": {
        "root": {
          "type": "string"
        },
        "compiler_find": {
          "type": "boolean",
          "description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror by running ``spack mirror add``"
        },
        "env": {
          "$ref": "#definitions/env",
          "description": "Manage spack environments via ``spack env`` command"
        },
        "install": {
          "$ref": "#definitions/install",
          "description": "Install spack packages by running ``spack install``. "
        },
        "verify_spack": {
          "type": "boolean",
          "description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
          "default": true
        },
        "test": {
          "$ref": "#definitions/test",
          "description": "Entry point to ``spack test``"
        }
      }
    }
  },
  "definitions": {
    "env": {
      "additionalProperties": false,
      "type": "object",
      "description": "Used for managing spack environment using ``spack env`` command. ",
      "properties": {
        "create": {
          "additionalProperties": false,
          "description": "Create a spack environment via ``spack env create``",
          "type": "object",
          "properties": {
            "remove_environment": {
              "type": "boolean",
              "description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
              "default": false
            },
            "name": {
              "type": "string",
              "description": "Name of spack environment to create"
            },
            "manifest": {
              "type": "string",
              "description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env create`` command"
            },
            "dir": {
              "type": "string",
              "description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
            }
          }
        },
        "activate": {
          "additionalProperties": false,
          "type": "object",
          "description": "Activate a spack environment via ``spack env activate``",
          "properties": {
            "name": {
              "type": "string",
              "description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env activate`` command"
            },
            "dir": {
              "type": "string",
              "description": "Activate spack environment from directory."
            }
          }
        },
        "rm": {
          "additionalProperties": false,
          "description": "Remove an existing spack environment via ``spack env rm``.",
          "type": "object",
          "required": [
            "name"
          ],
          "properties": {
            "name": {
              "type": "string",
              "description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
            }
          }
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror in spack environment by running ``spack mirror add``"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
        },
        "concretize": {
          "type": "boolean",
          "description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
        }
      }
    },
    "install": {
      "description": "Install spack packages using ``spack install`` command",
      "additionalProperties": false,
      "type": "object",
      "properties": {
        "options": {
          "type": "string",
          "description": "Pass options to ``spack install`` command"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "List of specs to install using ``spack install`` command"
        }
      }
    },
    "test": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "run"
      ],
      "properties": {
        "remove_tests": {
          "type": "boolean",
          "description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
        },
        "run": {
          "description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
          "type": "object",
          "required": [
            "specs"
          ],
          "additionalProperties": false,
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test run``"
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "List of specs to run tests by running ``spack test run <specs>``."
            }
          }
        },
        "results": {
          "type": "object",
          "description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
          "additionalProperties": false,
          "anyOf": [
            {
              "required": [
                "specs"
              ]
            },
            {
              "required": [
                "suite"
              ]
            },
            {
              "required": [
                "specs",
                "suite"
              ]
            }
          ],
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test results``"
            },
            "suite": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "Report results by  suite name by running ``spack test results <suite>``."
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "Report result by spec name by running ``spack test run -- <specs>``."
            }
          }
        }
      }
    }
  }
}