Buildtest Schemas

buildtest uses JSON Schema for validating buildspecs and buildtest configuration file. In this section, we will show how you can query the json schema and its example files using the command line tool buildtest schema.

Schema Naming Convention

All schema files use the file extension .schema.json to distinguish itself as a json schema definition from an ordinary json file. The schema files are located in buildtest/schemas directory.

CLI for buildtest schemas (buildtest schema)

You can use buildtest schema command to see the list of schemas supported by buildtest. The schema files are denoted by .schema.json file extension.

buildtest schema
$ buildtest schema
global.schema.json
definitions.schema.json
settings.schema.json
spack.schema.json
script.schema.json

Shown below is the command usage of buildtest schema

buildtest schema --help
$ buildtest schema --help
usage: buildtest [options] [COMMANDS] schema [-h] [-e] [-j] [-n Schema Name]

optional arguments:
  -h, --help            show this help message and exit
  -e, --example         Show schema examples
  -j, --json            Display json schema file
  -n Schema Name, --name Schema Name
                        show schema by name (e.g., script)

You must use the --name option to select a schema, for instance if you want to view the JSON Schema for script.schema.json you can run the following:

buildtest schema --name script.schema.json --json

Schema Files

Definition Schema

This schema is used for declaring definitions that need to be reused in multiple schemas. We use $ref keyword to reference definitions from this file.

definitions.schema.json
{
  "$id": "definitions.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "JSON Schema Definitions File. ",
  "description": "This file is used for declaring definitions that are referenced from other schemas",
  "definitions": {
    "list_of_strings": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
    "string_or_list": {
      "oneOf": [
        {
          "type": "string"
        },
        {
          "$ref": "#/definitions/list_of_strings"
        }
      ]
    },
    "list_of_ints": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "integer"
      }
    },
    "list_of_positive_integers": {
      "type": "array",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "integer",
        "minimum": 1
      }
    },
    "int_or_list": {
      "oneOf": [
        {
          "type": "integer"
        },
        {
          "$ref": "#/definitions/list_of_ints"
        }
      ]
    },
    "file_regex": {
      "type": "array",
      "description": "Specify a list of regular expressions to match files in the current working directory. The regular expression is matched using ``re.search`` python module. ",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "file",
          "exp"
        ],
        "properties": {
          "file": {
              "type": "string",
              "description": "Specify a file name to match with regular expression"
          },
          "exp": {
              "type": "string",
              "description": "Specify a regular expression to run on the selected file name"
          }
        }
      }
    },
    "file_regex_in_metrics": {
      "type": "object",
      "additionalProperties": false,
      "description": "Specify a regular expressions on a filepath used for assigning value to metrics. The regular expression is matched using ``re.search`` python module. ",
      "required": [
        "file",
        "exp"
      ],
      "properties": {
        "additionalProperties": false,
        "file": {
            "type": "string",
            "description": "Specify a file name to match with regular expression"
        },
        "exp": {
            "type": "string",
            "description": "Specify a regular expression to run on the selected file name"
        },
        "item": {
          "type": "integer",
          "minimum": 0,
          "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
        }
      }
    },
    "regex": {
      "type": "object",
      "additionalProperties": false,
      "description": "Perform regular expression search using ``re.search`` python module on stdout/stderr stream for reporting if test ``PASS``. ",
      "required": [
        "exp",
        "stream"
      ],
      "properties": {
        "stream": {
          "type": "string",
          "enum": [
            "stdout",
            "stderr"
          ],
          "description": "The stream field can be stdout or stderr. buildtest will read the output or error stream after completion of test and check if regex matches in stream"
        },
        "exp": {
          "type": "string",
          "description": "Specify a regular expression to run with input stream specified by ``stream`` field. buildtest uses re.search when performing regex"
        },
        "item": {
          "type": "integer",
          "minimum": 0,
          "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
        }
      }
    },
    "env": {
      "type": "object",
      "description": "One or more key value pairs for an environment (key=value)",
      "minItems": 1,
      "items": {
        "type": "object",
        "minItems": 1,
        "propertyNames": {
          "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"
        }
      }
    },
    "description": {
      "type": "string",
      "description": "The ``description`` field is used to document what the test is doing",
      "maxLength": 80
    },
    "summary": {
      "type": "string",
      "description": "The ``summary`` field is used to document what the test is doing and can be a multi-line string"
    },
    "tags": {
      "description": "Classify tests using a tag name, this can be used for categorizing test and building tests using ``--tags`` option",
      "$ref": "#/definitions/string_or_list"
    },
    "skip": {
      "type": "boolean",
      "description": "The ``skip`` is a boolean field that can be used to skip tests during builds. By default buildtest will build and run all tests in your buildspec file, if ``skip: True`` is set it will skip the buildspec."
    },
    "executor": {
      "type": "string",
      "description": "Select one of the executor name defined in your configuration file (``config.yml``). Every buildspec must have an executor which is responsible for running job. "
    },
    "needs": {
      "description": "A list of test names that are dependency before runnning job",
      "type": "array",
      "items": [
        {
          "oneOf": [
            { "type": "string" },
            {
              "type": "object",
              "patternProperties": {
                "^.*$": {
                  "additionalProperties": false,
                  "properties": {
                    "state": {"$ref": "#/definitions/state"},
                    "returncode": {"$ref": "#/definitions/returncode"}
                  }
                }
              }
            }
          ]
        }
      ]
    },
    "metrics_field": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "type"
      ],
      "properties": {
        "type": {
          "type": "string",
          "description": "Specify python data-type (str, int, float) to convert metric. ",
          "enum": [
            "str",
            "int",
            "float"
          ]
        },
        "regex": {
          "$ref": "#/definitions/regex"
        },
        "file_regex": {
          "$ref": "#/definitions/file_regex_in_metrics"
        }
      },
        "oneOf": [
          { "required": ["regex" ] },
          { "required": [ "file_regex"] }
        ]
    },
    "metrics": {
      "type": "object",
      "description": "This field is used for defining one or more metrics that is recorded for each test. A metric must have a unique name which is recorded in the test metadata.",
      "propertyNames": {
        "pattern": "^[A-Za-z0-9_.-]+$"
      },
      "additionalProperties": {
        "$ref": "#/definitions/metrics_field"
      }
    },
    "state": {
      "type": "string",
      "description": "explicitly mark state of test regardless of status calculation",
      "enum": [
        "PASS",
        "FAIL"
      ]
    },
    "returncode": {
      "description": "Specify a list of returncodes to match with script's exit code. buildtest will PASS test if script's exit code is found in list of returncodes. You must specify unique numbers as list and a minimum of 1 item in array",
      "$ref": "#/definitions/int_or_list"
    },
    "status": {
      "type": "object",
      "description": "The status section describes how buildtest detects PASS/FAIL on test. By default returncode 0 is a PASS and anything else is a FAIL, however buildtest can support other types of PASS/FAIL conditions.",
      "additionalProperties": false,
      "properties": {
        "slurm_job_state": {
          "type": "string",
          "enum": [
            "COMPLETED",
            "FAILED",
            "OUT_OF_MEMORY",
            "TIMEOUT"
          ],
          "description": "This field can be used to pass test based on Slurm Job State, if there is a match buildtest will report as ``PASS`` "
        },
        "pbs_job_state":
        {
          "type": "string",
          "description": "This field can be used to pass test based on PBS Job State, if there is a match buildtest will report as ``PASS`` ",
          "enum": [
            "H",
            "S",
            "F"
          ]
        },
        "lsf_job_state": {
          "type": "string",
          "description": "This field can be used to pass test based on LSF Job State, if there is a match buildtest will report as ``PASS`` ",
          "enum": [
            "DONE",
            "EXIT"
          ]
        },
        "returncode": { "$ref": "#/definitions/returncode" },
        "regex": {
          "$ref": "#/definitions/regex",
          "description": "Determine state (PASS/FAIL) of test based on regular expression on output or error stream"
        },
        "file_regex": {
          "$ref": "#/definitions/file_regex",
          "description": "Perform a regular expression search on a list of arbitrary files, if expression matches for any file buildtest will report as ``PASS``. This field is useful for checking output of a test that is not captured in stdout or stderr. "
        },

        "runtime": {
          "type": "object",
          "description": "The runtime section will pass test based on min and max values and compare with actual runtime. ",
          "properties": {
            "min": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a minimum runtime in seconds. The test will PASS if actual runtime exceeds min time."
            },
            "max": {
              "type": "number",
              "minimum": 0,
              "description": "Specify a maximum runtime in seconds. The test will PASS if actual runtime is less than max time"
            }
          }
        },
        "assert_ge": {
          "description": "Perform assertion of greater and equal (>=) with reference value",
          "$ref": "#/definitions/status_check_ge_gt_le_lt"
        },
        "assert_le": {
          "description": "Perform assertion of greater and equal (>=) with reference value",
          "$ref": "#/definitions/status_check_ge_gt_le_lt"
        },
        "assert_gt": {
          "description": "Perform assertion of greater than (>) with reference value",
          "$ref": "#/definitions/status_check_ge_gt_le_lt"
        },
        "assert_lt": {
          "description": "Perform assertion of less than (<) with reference value",
          "$ref": "#/definitions/status_check_ge_gt_le_lt"
        },
        "assert_eq":
        {
          "description": "Perform assertion of equality (=) with reference value",
          "$ref": "#/definitions/status_check_eq_ne"
        },
        "assert_ne":
        {
          "description": "Perform assertion of not equal (!=) with reference value",
          "$ref": "#/definitions/status_check_eq_ne"
        },
        "assert_range": {
          "type": "object",
          "description": "Perform assertion based on lower and upper bound",
          "additionalProperties": false,
          "required": ["comparisons"],
          "properties": {
            "mode": {
              "$ref": "#/definitions/mode"
            },
            "comparisons": {
              "type": "array",
              "items": {
                "type": "object",
                "additionalProperties": false,
                "required": [
                  "name",
                  "lower",
                  "upper"
                ],
                "properties": {
                  "name": {
                    "type": "string",
                    "description": "Name of metric to use for comparison"
                  },
                  "lower": {
                    "description": "Specify reference value for lower bound",
                    "type": "number"
                  },
                  "upper": {
                    "description": "Specify reference value for upper bound",
                    "type": "number"
                  }
                }
              }
            }
          }
        },
        "contains": {
          "description": "Check if metric value is in a list of reference values",
          "$ref": "#/definitions/status_check_contains_not_contains"
        },
        "not_contains": {
          "description": "Check if metric value not in a list of reference values ",
          "$ref": "#/definitions/status_check_contains_not_contains"
        },
        "is_symlink": {
          "description": "Check for list of files or directory paths that are symbolic links",
          "$ref": "#/definitions/list_of_strings"
        },
        "exists": {
          "description": "Check for list of file or directory path for existences using os.path.exists",
          "$ref": "#/definitions/list_of_strings"
        },
        "is_dir": {
          "description": "Check for list of filepaths are directories",
          "$ref": "#/definitions/list_of_strings"
        },
        "is_file": {
          "description": "Check for list of filepaths are files",
          "$ref": "#/definitions/list_of_strings"
        },
        "file_count": {
          "description": "Perform assertion check by comparing file count in a directory",
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "dir",
              "count"
            ],
            "properties": {
              "dir": {
                "type": "string",
                "description": "Directory path to check for file count"
              },
              "count": {
                "type": "integer",
                "description": "Number of files expected in directory",
                "minimum": 0
              },
              "depth": {
                "type": "integer",
                "description": "Depth of directory to search for files",
                "minimum": 0
              },
              "ext": {
                "$ref": "#/definitions/string_or_list",
                "description": "File extension to filter files in directory"
              },
              "filepattern": {
                "type": "string",
                "description": "Specify a regular expression when searching for files in directory"
              },
              "filetype": {
                "type": "string",
                "description": "Specify file type when searching for files in directory. It can be 'file', 'dir' or 'symlink'",
                "enum": [
                  "file",
                  "dir",
                  "symlink"
                ]
              },
              "file_traverse_limit": {
                "type": "integer",
                "description": "Limit the number of files to traverse when searching for files in directory",
                "minimum": 1,
                "default": 10000,
                "maximum": 999999
              }
            }
          }
        },
        "state": {
          "$ref": "#/definitions/state",
          "description": "explicitly mark state of test regardless of status calculation"
        },
        "mode": {
          "$ref": "#/definitions/mode"
        }
      }
    },
    "status_check_ge_gt_le_lt": {
       "type": "object",
        "additionalProperties": false,
        "required": ["comparisons"],
        "properties": {
          "mode": {
            "$ref": "#/definitions/mode"
          },
          "comparisons": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "name",
                "ref"
              ],
              "properties": {
                "name": {
                  "type": "string",
                  "description": "Name of metric to use for comparison"
                },
                "ref": {
                  "type": "number",
                  "description": "Specify reference value (int,float) for comparison"
                }
              }
            }
          }
        }
    },
    "status_check_eq_ne": {
       "type": "object",
        "additionalProperties": false,
        "required": ["comparisons"],
        "properties": {
          "mode": {
            "$ref": "#/definitions/mode"
          },
          "comparisons": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "name",
                "ref"
              ],
              "properties": {
                "name": {
                  "type": "string",
                  "description": "Name of metric to use for comparison"
                },
                "ref": {
                  "description": "Specify reference value (str, int, float) for comparison",
                  "oneOf": [
                    {
                      "type": "number"
                    },
                    {
                      "type": "string"
                    }
                  ]
                }
              }
            }
          }
        }
    },
    "status_check_contains_not_contains": {
      "type": "object",
      "required": ["comparisons"],
      "additionalProperties": false,
      "properties": {
        "mode": {
          "$ref": "#/definitions/mode"
        },
        "comparisons": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "name",
              "ref"
            ],
            "properties": {
              "name": {
                "type": "string",
                "description": "Name of metric to use for comparison"
              },
              "ref": {
                "type": "array",
                "minItems": 1,
                "description": "Specify a list of reference value"
              }
            }
          }
        }
      }
    },
    "mode": {
      "description": "Determine how the status check is resolved, for instance it can be logical AND or OR",
      "type": "string",
      "enum": ["or", "and", "OR", "AND"]
    },
    "BB": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Create burst buffer space, this specifies #BB options in your test."
    },
    "DW": {
      "$ref": "#/definitions/list_of_strings",
      "description": "Specify Data Warp option (#DW) when using burst buffer."
    },
    "sbatch": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #SBATCH options in test script."
    },
    "bsub": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #BSUB options in test script."
    },
    "cobalt": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #COBALT options in test script."
    },
    "pbs": {
      "$ref": "#/definitions/list_of_strings",
      "description": "This field is used for specifying #PBS directives in test script."
    },
    "executors": {
      "type": "object",
      "description": "Define executor specific configuration",
      "patternProperties": {
        "description": "Name of executor to override configuration",
        "^.*$": {
          "additionalProperties": false,
          "properties": {
            "env": {
              "$ref": "#/definitions/env"
            },
            "vars": {
              "$ref": "#/definitions/env"
            },
            "sbatch": {
              "$ref": "#/definitions/list_of_strings"
            },
            "bsub": {
              "$ref": "#/definitions/list_of_strings"
            },
            "pbs": {
              "$ref": "#/definitions/list_of_strings"
            },
            "cobalt": {
              "$ref": "#/definitions/list_of_strings"
            },
            "BB": {
              "$ref": "#/definitions/BB"
            },
            "DW": {
              "$ref": "#/definitions/DW"
            },
            "status": {
              "$ref": "#/definitions/status"
            },
            "metrics": {
              "$ref": "#/definitions/metrics"
            }
          }
        }
      }
    },
    "cc": {
      "type": "string",
      "description": "Set C compiler wrapper"
    },
    "fc": {
      "type": "string",
      "description": "Set Fortran compiler wrapper"
    },
    "cxx": {
      "type": "string",
      "description": "Set C++ compiler wrapper"
    },
    "cflags": {
      "type": "string",
      "description": "Set C compiler flags."
    },
    "fflags": {
      "type": "string",
      "description": "Set Fortran compiler flags."
    },
    "cxxflags": {
      "type": "string",
      "description": "Set C++ compiler flags."
    },
    "ldflags": {
      "type": "string",
      "description": "Set linker flags"
    },
    "cppflags": {
      "type": "string",
      "description": "Set C or C++ preprocessor flags"
    },
    "run": {
      "type": "string",
      "description": "Specify a series of commands to run."
    },
    "module": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
         "purge": {
           "type": "boolean",
            "description": "Run ``module purge`` if purge is set"
         },
         "load": {
           "$ref": "definitions.schema.json#/definitions/list_of_strings",
           "description": "Load one or more modules via ``module load``"
         },
         "restore": {
           "description": "Load a collection name via ``module restore``",
           "type": "string"
         },
         "swap": {
           "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
           "type": "array",
           "uniqueItems": true,
           "minItems": 2,
           "maxItems": 2,
           "items": {
             "type": "string"
           }
         }
       }
    }
  }
}

Settings Schema

This schema defines how buildtest configuration file is validated.

settings.schema.json
{
  "$id": "settings.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "buildtest configuration schema",
  "type": "object",
  "required": [
    "system"
  ],
  "additionalProperties": false,
  "properties": {
    "system": {
      "type": "object",
      "patternProperties": {
        "^.*$": {
          "$ref": "#/definitions/system"
        }
      }
    }
  },
  "definitions": {
    "terse": {
      "type": "boolean",
      "description": "A boolean to determine whether to enable terse mode"
    },
    "format": {
        "type": "string",
        "description": "Determine the format fields to display when viewing table results"
    },
    "count": {
      "type": "integer",
      "minimum": 1,
      "description": "Determine number of records to display in a table"
    },

    "system": {
      "required": [
        "executors",
        "moduletool",
        "hostnames",
        "compilers",
        "buildspecs",
        "report"
      ],
      "additionalProperties": false,
      "type": "object",
      "properties": {
        "hostnames": {
          "type": "array",
          "description": "Specify a list of hostnames to check where buildtest can run for the given system record",
          "items": {
            "type": "string"
          }
        },
        "description": {
          "type": "string",
          "description": "system description field"
        },
        "poolsize":
        {
          "type": "integer",
          "minimum": 1,
          "description": "Specify size of Process Pool for parallel processing using ``multiprocessing.Pool``"
        },
        "max_jobs": {
            "type": "integer",
            "minimum": 1,
            "description": "Maximum number of jobs that can be run at a given time for a particular executor"
        },
        "testdir": {
          "type": "string",
          "description": "Specify full path to test directory where buildtest will write tests."
        },
        "logdir": {
          "type": "string",
          "description": "Specify location where buildtest will write log files"
        },
        "moduletool": {
          "type": "string",
          "description": "Specify modules tool used for interacting with ``module`` command. ",
          "enum": [
            "environment-modules",
            "lmod",
            "none"
          ]
        },
        "timeout": {
          "type": "integer",
          "minimum": 1,
          "description": "Specify timeout duration in number of seconds"
        },
        "pager":
          {
            "type": "boolean",
            "description": "A boolean to determine whether to enable paging when viewing buildspec cache"
          },
        "buildspecs":
        {
          "type": "object",
          "description": "Specify configuration for ``buildtest buildspec`` command",
          "additionalProperties": false,
          "properties": {
            "rebuild": {
              "type": "boolean",
              "description": "A boolean to determine whether to rebuild buildspec cache"
            },
            "count": { "$ref": "#/definitions/count" } ,
            "format": {
              "$ref": "#/definitions/format"
            },
            "terse": { "$ref": "#/definitions/terse" },
            "root": {
              "type": "array",
              "items": {
                "type": "string"
              },
              "description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
            }
          }
        },
        "report":
        {
          "type": "object",
          "additionalProperties": false,
          "description": "Specify configuration for ``buildtest report`` command",
          "properties": {
            "terse": { "$ref": "#/definitions/terse" },
            "format": { "$ref": "#/definitions/format" },
            "count": { "$ref": "#/definitions/count" }
          }
        },
        "processor": {
          "type": "object",
          "description": "Specify processor information",
          "additionalProperties": false,
          "properties": {
            "numcpus": {"type": "integer", "minimum": 1, "description": "Specify Total Number of CPUs"},
            "sockets": {"type": "integer", "minimum": 1, "description": "Specify Number of CPU Sockets"},
            "cores": {"type": "integer", "minimum": 1, "description": "Specify Number of Physical Cores"},
            "threads_per_core": {"type": "integer", "minimum":  1, "description": "Specify Threads per Core" },
            "core_per_socket": {"type": "integer", "minimum": 1, "description": "Specify Cores per Socket"},
            "model": {"type": "string", "description": "Specify Processor Model" },
            "arch": {"type": "string", "description": "Specify processor architecture"},
            "vendor": {"type": "string", "description": "Vendor Name"}
          }
        },
        "compilers": {
          "type": "object",
          "description": "Declare compiler section for defining system compilers that can be referenced in buildspec.",
          "additionalProperties": false,
          "properties": {
            "enable_prgenv": {
              "type": "boolean",
              "description": "Enable support for Programming Environment"
            },
            "modulepath": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
            "purge": {"type": "boolean", "description": "A boolean to determine whether to purge modules via ``module purge`` when generating compiler declaration"},
            "prgenv_modules": {
              "type": "object",
              "additionalProperties": false,
              "properties": {
                "gcc": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for gcc",
                  "enum": [
                    "PrgEnv-gnu"
                  ]
                },
                "intel": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for intel",
                  "enum": [
                    "PrgEnv-intel"
                  ]
                },
                "cray": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for cray",
                  "enum": [
                    "PrgEnv-cray"
                  ]
                },
                "nvhpc": {
                  "type": "string",
                  "description": "Specify name of Programming Environment module for nvhpc",
                  "enum": [
                    "PrgEnv-nvhpc",
                    "PrgEnv-nvidia"
                  ]
                }
              }
            },
            "find": {
              "type": "object",
              "additionalProperties": false,
              "description": "Find compilers by specifying regular expression that is applied to modulefile names",
              "properties": {
                "gcc": {
                  "type": "string",
                  "description": "Specify a regular expression to search for gcc compilers from your module stack"
                },
                "intel": {
                  "type": "string",
                  "description": "Specify a regular expression to search for intel compilers from your module stack"
                },
                "cray": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cray compilers from your module stack"
                },
                "clang": {
                  "type": "string",
                  "description": "Specify a regular expression to search for clang compilers from your module stack"
                },
                "cuda": {
                  "type": "string",
                  "description": "Specify a regular expression to search for cuda compilers from your module stack"
                },
                "pgi": {
                  "type": "string",
                  "description": "Specify a regular expression to search for pgi compilers from your module stack"
                },
                "upcxx": {
                  "type": "string",
                  "description": "Specify a regular expression to search for upcxx compilers from your module stack"
                },
                "nvhpc": {
                  "type": "string",
                  "description": "Specify a regular expression to search for nvhpc compilers from your module stack"
                }
              }
            },
            "compiler": {
              "type": "object",
              "additionalProperties": false,
              "description": "Start of compiler declaration",
              "properties": {
                "gcc": {
                  "description": "Declaration of one or more GNU compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "intel": {
                  "description": "Declaration of one or more Intel compilers. ",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "cray": {
                  "description": "Declaration of one or more Cray compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "pgi": {
                  "description": "Declaration of one or more PGI compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "clang": {
                  "description": "Declaration of one or more Clang compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "cuda": {
                  "description": "Declaration of one or more CUDA compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "upcxx": {
                  "description": "Declaration of one or more UPCXX compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                },
                "nvhpc": {
                  "description": "Declaration of one or more NVHPC compilers.",
                  "type": "object",
                  "patternProperties": { "^.*$": { "$ref": "#/definitions/compiler_section" } }
                }
              }
            }
          }
        },
        "executors": {
            "type": "object",
            "additionalProperties": false,
            "description": "The executor section is used for declaring your executors that are responsible for running jobs. The executor section can be ``local``, ``lsf``, ``slurm``, ``cobalt``. The executors are referenced in buildspec using ``executor`` field.",
            "required": ["local"],
            "properties": {
              "defaults": {
                "type": "object",
                "description": "Specify default executor settings for all executors",
                "additionalProperties": false,
                "properties": {
                  "pollinterval": {
                    "type": "integer",
                    "description": "Specify poll interval in seconds after job submission, where buildtest will sleep and poll all jobs for job states. This field can be configured based on your preference. Excessive polling every few seconds can result in system degradation. ",
                    "minimum": 1,
                    "default": 30
                  },
                  "maxpendtime": {
                    "$ref": "#/definitions/maxpendtime"
                  },
                  "account": {
                    "$ref": "#/definitions/account"
                  }
                }
              },
              "local": {
                "type": "object",
                "description": "The ``local`` section is used for declaring local executors for running jobs on local machine",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/local"
                },
                "minProperties": 1
              },
              "lsf": {
                "type": "object",
                "description": "The ``lsf`` section is used for declaring LSF executors for running jobs using LSF scheduler",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/lsf"
                }
              },
              "slurm": {
                "type": "object",
                "description": "The ``slurm`` section is used for declaring Slurm executors for running jobs using Slurm scheduler",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/slurm"
                }
              },
              "cobalt": {
                "type": "object",
                "description": "The ``cobalt`` section is used for declaring Cobalt executors for running jobs using Cobalt scheduler",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/cobalt"
                }
              },
              "pbs": {
                "type": "object",
                "description": "The ``pbs`` section is used for declaring PBS executors for running jobs using PBS scheduler",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/pbs"
                }
              },
              "container": {
                "type": "object",
                "description": "The ``container`` section is used for declaring container executors for running jobs using container technology",
                "propertyNames": {
                  "pattern": "^[A-Za-z0-9_.-]+$"
                },
                "additionalProperties": {
                  "$ref": "#/definitions/container"
                }
              }
            }
          },
        "cdash": {
          "type": "object",
          "description": "Specify CDASH configuration used to upload tests via 'buildtest cdash' command",
          "required": ["url", "project", "site"],
          "properties": {
            "url": {
              "type": "string",
              "description": "Url to CDASH server"
            },
            "project": {
              "type": "string",
              "description": "Name of CDASH project"
            },
            "site": {
              "type": "string",
              "description": "Site Name reported in CDASH"
            }
          }
        },
        "profiles": {
          "type": "object",
          "description": "The profiles section is used for declaring one or more profiles that can be used to run ``buildtest build`` that are captured as command options ",
            "propertyNames": {
                "pattern": "^[A-Za-z0-9_.-]+$"
            },
            "additionalProperties": {
              "$ref": "#/definitions/profiles"
            }
        }
      }
    },
    "cc": {
      "description": "Specify path to C compiler wrapper. You may specify a compiler wrapper such as ``gcc`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper.",
      "type": "string"
    },
    "cxx": {
      "type": "string",
      "description": "Specify path to C++ compiler wrapper. You may specify a compiler wrapper such as ``g++`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "fc": {
      "type": "string",
      "description": "Specify path to Fortran compiler wrapper. You may specify a compiler wrapper such as ``gfortran`` assuming its in $PATH or you can use ``modules`` property to resolve path to compiler wrapper."
    },
    "compiler_section": {
      "description": "A compiler section is composed of ``cc``, ``cxx`` and ``fc`` wrapper these are required when you need to specify compiler wrapper.",
      "type": "object",
      "additionalProperties": false,
      "required": [ "cc",  "cxx",  "fc" ],
      "properties": {
        "cc": { "$ref": "#/definitions/cc" },
        "cxx": { "$ref": "#/definitions/cxx" },
        "fc": { "$ref": "#/definitions/fc" },
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "unique_string_array": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string"
      }
    },
    "disable": {"type": "boolean", "description": "Disable an executor"},
    "module": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "purge": {
          "type": "boolean",
          "description": "Run ``module purge`` if purge is set"
        },
        "load": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Load one or more modules via ``module load``"
        },
        "swap": {
          "description": "Swap modules using ``module swap``. The swap property expects 2 unique modules.",
          "type": "array",
          "uniqueItems": true,
          "minItems": 2,
          "maxItems": 2,
          "items": { "type": "string" }
        },
         "restore": {
           "description": "Load a collection name via ``module restore``",
           "type": "string"
         }
      }
    },
    "script": {
      "type": "array",
      "additionalProperties": false,
      "items": { "type": "string" }
    },
    "maxpendtime": {
      "type": "integer",
      "description": "Cancel job if it is still pending in queue beyond maxpendtime",
      "minimum": 1,
      "default": 86400
    },
    "account": {
      "type": "string",
      "description": "Specify Job Account for charging resources"
    },
    "local": {
      "type": "object",
      "description": "An instance object of local executor",
      "additionalProperties": false,
      "required": [ "shell" ],
      "properties": {
        "description": { "$ref": "#/definitions/description" },
        "shell": {
          "type": "string",
          "description": "Specify the shell launcher you want to use when running tests locally"
        },
        "before_script": { "#ref": "#/definitions/script" },
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "slurm": {
      "type": "object",
      "additionalProperties": false,
      "description": "An instance object of slurm executor",
      "properties": {
        "description": { "$ref": "#/definitions/description" },
        "launcher": {
          "type": "string",
          "enum": [ "sbatch" ],
          "description": "Specify the slurm batch scheduler to use. This overrides the default ``launcher`` field. This must be ``sbatch``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any other options for ``sbatch`` used by this executor for running all jobs."
        },
        "cluster": {
          "type": "string",
          "description": "Specify the slurm cluster you want to use ``-M <cluster>``"
        },
        "partition": {
          "type": "string",
          "description": "Specify the slurm partition you want to use ``-p <partition>``"
        },
        "qos": {
          "type": "string",
          "description": "Specify the slurm qos you want to use ``-q <qos>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "lsf": {
      "type": "object",
      "description": "An instance object of lsf executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": {
          "type": "string",
          "description": "description field for documenting your executor"
        },
        "launcher": {
          "type": "string",
          "enum": [ "bsub" ],
          "description": "Specify the lsf batch scheduler to use. This overrides the default ``launcher`` field. It must be ``bsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``bsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "cobalt": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": { "$ref": "#/definitions/description" },
        "launcher": {
          "type": "string",
          "enum": [ "qsub" ],
          "description": "Specify the cobalt batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "pbs": {
      "type": "object",
      "description": "An instance object of cobalt executor",
      "additionalProperties": false,
      "required": [ "queue" ],
      "properties": {
        "description": { "$ref": "#/definitions/description" },
        "launcher": {
          "type": "string",
          "enum": [ "qsub" ],
          "description": "Specify the pbs batch scheduler to use. This overrides the default ``launcher`` field. It must be ``qsub``. "
        },
        "options": {
          "type": "array",
          "items": { "type": "string" },
          "description": "Specify any options for ``qsub`` for this executor when running all jobs associated to this executor"
        },
        "queue": {
          "type": "string",
          "description": "Specify the lsf queue you want to use ``-q <queue>``"
        },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "maxpendtime": {
          "description": "overrides default ``maxpendtime`` value",
          "$ref": "#/definitions/maxpendtime"
        },
        "account": {
          "description": "overrides default ``account`` value",
          "$ref": "#/definitions/account"
        },
        "disable": {"$ref":  "#/definitions/disable"},
        "module": { "$ref": "#/definitions/module" }
      }
    },
    "container": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "description": { "$ref": "#/definitions/description" },
        "before_script": {
          "description": "The ``before_script`` section can be used to specify commands before start of test. The script will be sourced in active shell.",
          "#ref": "#/definitions/script"
        },
        "platform": {
          "type": "string",
          "description": "Specify container platform to use for container. ",
          "enum": [
            "docker",
            "singularity",
            "podman"
          ]
        },
        "image": {
          "type": "string",
          "description": "Specify container image to use for container. "
        },
        "mounts": {
          "type": "string",
          "description": "Specify a list of directory paths to bind mount into the container"
        },
        "options": {
          "type": "string",
          "description": "Specify a list of options to pass to container runtime. "
        },
        "disable": {"$ref":  "#/definitions/disable"}
      }
    },
    "profiles":
    {
      "type": "object",
      "properties": {
        "buildspecs": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
        "exclude-buildspecs": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
        "tags": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
        "exclude-tags": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
        "executors": {"$ref": "definitions.schema.json#/definitions/list_of_strings"},
        "filter": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "type": {
              "type": "array",
              "description": "Specify the `type` field to determine which schema to use during filtering",
              "uniqueItems": true,
              "minItems": 1,
              "items": {
                "type": "string",
                 "enum": ["script", "spack"]
              }
            },
            "tags": {
              "description": "Filter by tags",
              "$ref": "definitions.schema.json#/definitions/list_of_strings"
            },
            "maintainers": {
              "description": "Filter by maintainers",
              "$ref": "definitions.schema.json#/definitions/list_of_strings"
            }
          }
        },
        "module": {"type": "string"},
        "unload-modules": {"type": "string"},
        "module-purge": {"type": "boolean"},
        "rebuild": {"type": "integer", "minimum": 1, "maximum": 50, "description": "Specify number of tests to rebuild"},
        "limit": {"type": "integer", "minimum": 1, "description": "Limit number of tests to build"},
        "account": {"$ref": "#/definitions/account"},
        "maxpendtime": { "$ref": "#/definitions/maxpendtime" },
        "pollinterval": {"type": "integer", "minimum": 1, "description": "Specify poll interval in seconds"},
        "procs": {"$ref": "definitions.schema.json#/definitions/list_of_positive_integers"},
        "nodes": {"$ref": "definitions.schema.json#/definitions/list_of_positive_integers"},
        "testdir": {"type": "string", "description": "Specify test directory where buildtest will write tests."},
        "timeout": {"type": "integer", "minimum": 1},
        "executor-type": {
          "type": "string",
          "description": "Specify the `executor-type` field used by ``--executor-type`` option which determines if test will run by local or batch executor",
          "enum": ["local", "batch"]
        },
        "remove-stagedir": {"type": "boolean", "description": "Remove stage directory after test completes"},
        "max-jobs": {"type": "integer", "minimum": 1, "description": "Maximum number of jobs that can be run at a given time for a particular executor"}
      }
    },
    "description": {
      "type": "string",
      "description": "description field for documenting your executor"
    }
  }
}

Global Schema

This schema is used for validating buildspec file and validates outer level structure of test. This is referred as Global Schema

global.schema.json
{
  "$id": "global.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "global schema",
  "description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas",
  "type": "object",
  "required": ["buildspecs"],
  "additionalProperties": false,
  "properties": {
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "maintainers": {
      "type": "array",
      "description": "One or more maintainers or aliases",
      "uniqueItems": true,
      "minItems": 1,
      "items": {
        "type": "string"
      }
    },
   "buildspecs": {
     "type": "object",
     "description": "This section is used to define one or more tests (buildspecs). Each test must be unique name",
     "propertyNames": {
       "pattern": "^[A-Za-z_.-][A-Za-z0-9_.-]*$",
       "maxLength": 48
     }
   }
  }
}

Script Schema

This is the script schema used for writing scripts (bash, csh, sh, zsh, tcsh, python) and this is used for validating test instance when type: script is specified. For more details on script schema see Script Schema.

script.schema.json
{
  "$id": "script.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "script schema version",
  "description": "The script schema is of ``type: script`` in sub-schema which is used for running shell scripts",
  "type": "object",
  "required": [
    "type",
    "run",
    "executor"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^script$",
      "description": "Select schema type to use when validating buildspec. This must be of set to 'script'"
    },
    "description": {
      "$ref": "definitions.schema.json#/definitions/description"
    },
    "summary": {
      "$ref": "definitions.schema.json#/definitions/summary"
    },
    "sbatch": {
      "$ref": "definitions.schema.json#/definitions/sbatch"
    },
    "bsub": {
      "$ref": "definitions.schema.json#/definitions/bsub"
    },
    "cobalt": {
      "$ref": "definitions.schema.json#/definitions/cobalt"
    },
    "pbs": {
      "$ref": "definitions.schema.json#/definitions/pbs"
    },
    "BB": {
      "$ref": "definitions.schema.json#/definitions/BB"
    },
    "DW": {
      "$ref": "definitions.schema.json#/definitions/DW"
    },
    "env": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "vars": {
      "$ref": "definitions.schema.json#/definitions/env"
    },
    "executor": {
      "$ref": "definitions.schema.json#/definitions/executor"
    },
    "needs": {
      "$ref": "definitions.schema.json#/definitions/needs"
    },
    "shell": {
      "type": "string",
      "description": "Specify a shell launcher to use when running jobs. This sets the shebang line in your test script. The ``shell`` key can be used with ``run`` section to describe content of script and how its executed"
    },
    "shebang": {
      "type": "string",
      "description": "Specify a custom shebang line. If not specified buildtest will automatically add it in the test script."
    },
    "run": {
      "$ref": "definitions.schema.json#/definitions/run"
    },
    "status": {
      "$ref": "definitions.schema.json#/definitions/status"
    },
    "skip": {
      "$ref": "definitions.schema.json#/definitions/skip"
    },
    "tags": {
      "$ref": "definitions.schema.json#/definitions/tags"
    },
    "metrics": {
      "$ref": "definitions.schema.json#/definitions/metrics"
    },
    "executors": {
      "$ref": "definitions.schema.json#/definitions/executors"
    },
    "compilers": {
      "type": "object",
      "required": [
        "name"
      ],
      "additionalProperties": false,
      "properties": {
        "name": {
          "description": "Specify a list of regular expression to search compiler instance from buildtest settings.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "exclude": {
          "description": "Specify a list of named compilers to exclude when building test based on regular expression specified in ``name`` property. The ``exclude`` property has no effect if named compiler not found based on regular expression.",
          "$ref": "definitions.schema.json#/definitions/list_of_strings"
        },
        "default": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "gcc": {
              "$ref": "#definitions/default_compiler_config"
            },
            "intel": {
              "$ref": "#definitions/default_compiler_config"
            },
            "pgi": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cray": {
              "$ref": "#definitions/default_compiler_config"
            },
            "clang": {
              "$ref": "#definitions/default_compiler_config"
            },
            "cuda": {
              "$ref": "#definitions/default_compiler_config"
            },
            "upcxx": {
              "$ref": "#definitions/default_compiler_config"
            },
            "nvhpc": {
              "$ref": "#definitions/default_compiler_config"
            }
          }
        },
        "config": {
          "type": "object",
          "description": "Specify compiler configuration based on named compilers.",
          "patternProperties": {
            "^.*$": {
              "$ref": "#definitions/compiler_declaration"
            }
          }
        }
      }
    },
    "container": {
      "type": "object",
      "required": [
          "image",
          "platform"
        ],
      "properties": {
        "platform": {
          "type": "string",
          "description": "Specify a container platform to use when running test. This is used for running commands inside container.",
          "enum": [
            "docker",
            "podman",
            "singularity"
          ]
        },
        "image": {
          "type": "string",
          "description": "Specify a container image to use when running test. This is used for running commands inside container."
        },
        "mounts": {
          "type": "string",
          "description": "Specify a list of directory paths to bind mount into the container"
        },
        "options": {
          "type": "string",
          "description": "Specify a list of options to pass to container runtime. "
        },
        "command": {
          "type": "string",
          "description": "Specify a list of commands to run inside container. This is used for running commands inside container."
        }
      }
    }
  },
  "definitions": {
    "compiler_declaration": {
      "type": "object",
      "description": "Specify compiler configuration at compiler level. The ``config`` section has highest precedence when searching compiler configuration. This overrides fields found in compiler group and ``all`` property",
      "additionalProperties": false,
      "properties": {
        "cc": {
          "$ref": "definitions.schema.json#/definitions/cc"
        },
        "fc": {
          "$ref": "definitions.schema.json#/definitions/fc"
        },
        "cxx": {
          "$ref": "definitions.schema.json#/definitions/cxx"
        },
        "cflags": {
          "$ref": "definitions.schema.json#/definitions/cflags"
        },
        "fflags": {
          "$ref": "definitions.schema.json#/definitions/fflags"
        },
        "cxxflags": {
          "$ref": "definitions.schema.json#/definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "definitions.schema.json#/definitions/ldflags"
        },
        "cppflags": {
          "$ref": "definitions.schema.json#/definitions/cppflags"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "run": {
          "$ref": "definitions.schema.json#/definitions/run"
        },
        "module": {
          "$ref": "definitions.schema.json#/definitions/module"
        }
      },
      "default_compiler_config": {
        "type": "object",
        "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
        "properties": {
          "cc": {
            "$ref": "definitions.schema.json#/definitions/cc"
          },
          "fc": {
            "$ref": "definitions.schema.json#/definitions/fc"
          },
          "cxx": {
            "$ref": "definitions.schema.json#/definitions/cxx"
          },
          "cflags": {
            "$ref": "definitions.schema.json#/definitions/cflags"
          },
          "fflags": {
            "$ref": "definitions.schema.json#/definitions/fflags"
          },
          "cxxflags": {
            "$ref": "definitions.schema.json#/definitions/cxxflags"
          },
          "ldflags": {
            "$ref": "definitions.schema.json#/definitions/ldflags"
          },
          "cppflags": {
            "$ref": "definitions.schema.json#/definitions/cppflags"
          },
          "env": {
            "$ref": "definitions.schema.json#/definitions/env"
          },
          "vars": {
            "$ref": "definitions.schema.json#/definitions/env"
          },
          "run": {
            "$ref": "definitions.schema.json#/definitions/run"
          }
        }
      }
    },
    "default_compiler_config": {
      "type": "object",
      "description": "Specify compiler configuration for group of compilers. Use this property if you want to define common configuration for all compilers of same group. This property overrides ``all`` property. ",
      "properties": {
        "cc": {
          "$ref": "definitions.schema.json#/definitions/cc"
        },
        "fc": {
          "$ref": "definitions.schema.json#/definitions/fc"
        },
        "cxx": {
          "$ref": "definitions.schema.json#/definitions/cxx"
        },
        "cflags": {
          "$ref": "definitions.schema.json#/definitions/cflags"
        },
        "fflags": {
          "$ref": "definitions.schema.json#/definitions/fflags"
        },
        "cxxflags": {
          "$ref": "definitions.schema.json#/definitions/cxxflags"
        },
        "ldflags": {
          "$ref": "definitions.schema.json#/definitions/ldflags"
        },
        "cppflags": {
          "$ref": "definitions.schema.json#/definitions/cppflags"
        },
        "env": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "vars": {
          "$ref": "definitions.schema.json#/definitions/env"
        },
        "status": {
          "$ref": "definitions.schema.json#/definitions/status"
        },
        "run": {
          "$ref": "definitions.schema.json#/definitions/run"
        }
      }
    }
  }
}

Spack Schema

This schema is used for writing tests with spack package manager using type: spack field. For more details see Buildtest Spack Integration.

spack.schema.json
{
  "$id": "spack.schema.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "spack schema version",
  "description": "The spack schema is referenced using ``type: spack`` which is used for generating tests using spack package manager",
  "type": "object",
  "required": [
    "type",
    "executor",
    "spack"
  ],
  "additionalProperties": false,
  "properties": {
    "type": {
      "type": "string",
      "pattern": "^spack$",
      "description": "Select schema type to use when validating buildspec. This must be set to 'spack'"
    },
    "description": { "$ref": "definitions.schema.json#/definitions/description" },
    "summary": { "$ref": "definitions.schema.json#/definitions/summary" },
    "executor": { "$ref": "definitions.schema.json#/definitions/executor" },
    "env": { "$ref": "definitions.schema.json#/definitions/env" },
    "vars": { "$ref": "definitions.schema.json#/definitions/env" },
    "sbatch": { "$ref": "definitions.schema.json#/definitions/sbatch" },
    "bsub": { "$ref": "definitions.schema.json#/definitions/bsub" },
    "cobalt": { "$ref": "definitions.schema.json#/definitions/cobalt" },
    "pbs": { "$ref": "definitions.schema.json#/definitions/pbs" },
    "BB": { "$ref": "definitions.schema.json#/definitions/BB" },
    "DW": { "$ref": "definitions.schema.json#/definitions/DW" },
    "skip": { "$ref": "definitions.schema.json#/definitions/skip" },
    "tags": { "$ref": "definitions.schema.json#/definitions/tags" },
    "status": { "$ref": "definitions.schema.json#/definitions/status" },
    "metrics": { "$ref": "definitions.schema.json#/definitions/metrics" },
    "executors": { "$ref": "definitions.schema.json#/definitions/executors" },
    "pre_cmds": {
      "type": "string",
      "description": "Shell commands run before spack"
    },
    "post_cmds": {
      "type": "string",
      "description": "Shell commands run after spack"
    },
    "needs": {
      "$ref": "definitions.schema.json#/definitions/needs"
    },
    "spack": {
      "type": "object",
      "description": "Entry point to spack configuration",
      "additionalProperties": false,
      "properties": {
        "root": {
          "type": "string",
          "description": "Specify location for root of spack directory"
        },
        "compiler_find": {
          "type": "boolean",
          "description": "Run ``spack compiler find`` if set to ``True``. This is run right after sourcing spack startup script."
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror by running ``spack mirror add``"
        },
        "env": {
          "$ref": "#definitions/env",
          "description": "Manage spack environments via ``spack env`` command"
        },
        "install": {
          "$ref": "#definitions/install",
          "description": "Install spack packages by running ``spack install``. "
        },
        "load": {
          "$ref": "#definitions/load",
          "description": "load spack packages by running ``spack load`` "
        },
        "verify_spack": {
          "type": "boolean",
          "description": "This boolean will determine if we need to check for file existence where spack is cloned via ``root`` property and file **$SPACK_ROOT/share/spack/setup-env.sh** exists. These checks can be disabled by setting this to ``False`` which can be useful if you dont want buildtest to raise exception during test generation process and test is skipped.",
          "default": true
        },
        "test": {
          "$ref": "#definitions/test",
          "description": "Entry point to ``spack test``"
        }
      }
    }
  },
  "definitions": {
    "env": {
      "additionalProperties": false,
      "type": "object",
      "description": "Used for managing spack environment using ``spack env`` command. ",
      "properties": {
        "create": {
          "additionalProperties": false,
          "description": "Create a spack environment via ``spack env create``",
          "type": "object",
          "properties": {
            "remove_environment": {
              "type": "boolean",
              "description": "Remove existing spack environment before creating new environment. If set to ``True`` we will run ``spack env rm -y <name>``.",
              "default": false
            },
            "name": {
              "type": "string",
              "description": "Name of spack environment to create"
            },
            "manifest": {
              "type": "string",
              "description": "Specify path to spack manifest file (``spack.yaml`` or ``spack.lock``) when creating environment"
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env create`` command"
            },
            "dir": {
              "type": "string",
              "description": "Create a spack environment in a specific directory. This will run ``spack env create -d <dir>``. Directory path does not have to exist prior to execution however user must have appropriate ACL in-order to create directory."
            }
          }
        },
        "deactivate": {
          "type": "boolean",
          "description": "Deactivate a spack environment prior to activating it via ``spack env deactivate``"
        },
        "activate": {
          "additionalProperties": false,
          "type": "object",
          "description": "Activate a spack environment via ``spack env activate``",
          "properties": {
            "name": {
              "type": "string",
              "description": "Name of spack environment to activate. In order to activate spack environment ``my-project`` you need to run ``spack env activate my-project`` which is specified by ``name: my-project``."
            },
            "options": {
              "type": "string",
              "description": "Pass options to ``spack env activate`` command"
            },
            "dir": {
              "type": "string",
              "description": "Activate spack environment from directory."
            }
          }
        },
        "rm": {
          "additionalProperties": false,
          "description": "Remove an existing spack environment via ``spack env rm``.",
          "type": "object",
          "required": [
            "name"
          ],
          "properties": {
            "name": {
              "type": "string",
              "description": "Remove spack environment by name. This will run ``spack env rm -y <name>``."
            }
          }
        },
        "mirror": {
          "$ref": "definitions.schema.json#/definitions/env",
          "description": "Add mirror in spack environment by running ``spack mirror add``"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "Add specs to environment by running ``spack add <specs>``. The ``specs`` is a list of string which expect the argument to be name of spack package."
        },
        "concretize": {
          "type": "boolean",
          "description": "If ``concretize: true`` is set, we will concretize spack environment by running ``spack concretize -f`` otherwise this line will be ignored."
        }
      }
    },
    "install": {
      "description": "Install spack packages using ``spack install`` command",

      "type": "object",
      "properties": {
        "options": {
          "type": "string",
          "description": "Pass options to ``spack install`` command"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "List of specs to install using ``spack install`` command"
        }
      }
    },
    "load": {
      "description": "load spack packages using ``spack load`` command",

      "type": "object",
      "properties": {
        "options": {
          "type": "string",
          "description": "Pass options to ``spack load`` command"
        },
        "specs": {
          "$ref": "definitions.schema.json#/definitions/list_of_strings",
          "description": "List of specs to install using ``spack load`` command"
        }
      }
    },
    "test": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "run",
        "results"
      ],
      "properties": {
        "remove_tests": {
          "type": "boolean",
          "description": "Remove all test suites in spack before running test via ``spack test run``. If set to ``True`` we will run ``spack test remove -y`` which will remove all test suites."
        },
        "run": {
          "description": "Run tests using spack via ``spack test run`` command. This command requires specs are installed in your spack instance prior to running tests.",
          "type": "object",
          "required": [
            "specs"
          ],
          "additionalProperties": false,
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test run``"
            },
            "specs": {
              "$ref": "definitions.schema.json#/definitions/list_of_strings",
              "description": "List of specs to run tests by running ``spack test run <specs>``."
            }
          }
        },
        "results": {
          "type": "object",
          "description": "View test results via ``spack test results`` after running tests via ``spack test run``. Results can be viewed using suitename or installed specs or both.",
          "additionalProperties": false,
          "properties": {
            "option": {
              "type": "string",
              "description": "Pass options to ``spack test results``"
            },
            "specs": {
                  "$ref": "definitions.schema.json#/definitions/list_of_strings",
                  "description": "Report result by spec name by running ``spack test run -- <specs>``."
            }
          }
        }
      }
    }
  }
}

Schema Examples

The schema examples are great way to help write your buildspecs and help you understand the edge cases that can lead to an invalid buildspec. The schema examples are used in buildtest regression test for validating the schemas. We expose the examples through buildtest client so its accessible for everyone.

In order to view an example you can run:

buildtest schema -n <schema> --example

Settings Schema Examples

buildtest schema -n settings.schema.json --example
$ buildtest schema -n settings.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: N/A                                                             
    executors:                                                                  
      local:                                                                    
        # the line below is invalid due to pattern property                     
        ==bash:                                                                 
          description: submit jobs on local machine                             
          shell: bash -v                                                        
      slurm:                                                                    
        # the line below is invalid due to pattern property                     
        $haswell:                                                               
          launcher: sbatch                                                      
          options: ["-p haswell", "-t 00:10"]                                   
      lsf:                                                                      
        <>batch:                                                                
          launcher: bsub                                                        
          queue: batch                                                          
          options: ["-q batch", "-t 00:10"]                                     
      cobalt:                                                                   
        <normal>:                                                               
          launcher: qsub                                                        
          queue: normal                                                         
          options: ["-n 1", "-t 10"]                                            
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: none                                                            
    poolsize: 1                                                                 
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
    executors:                                                                  
      defaults:                                                                 
        pollinterval: 10                                                        
        maxpendtime: 30                                                         
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      pbs:                                                                      
        workq:                                                                  
          queue: workq                                                          
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    logdir: $BUILDTEST_ROOT/logs                                                
    testdir: $BUILDTEST_ROOT/tests                                              
    moduletool: none                                                            
    poolsize: 1                                                                 
    max_jobs: 1                                                                 
    pager: False                                                                
    cdash:                                                                      
      url: https://my.cdash.org                                                 
      project: buildtest                                                        
      site: laptop                                                              
    processor:                                                                  
      numcpus: 8                                                                
      cores: 4                                                                  
      threads_per_core: 2                                                       
      sockets: 1                                                                
      model: "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz"                         
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
    executors:                                                                  
      local:                                                                    
        bash:                                                                   
          description: submit jobs on local machine using bash shell            
          shell: bash                                                           
          module:                                                               
            purge: True                                                         
            restore: "Default"                                                  
            load: ["gcc"]                                                       
        sh:                                                                     
          description: submit jobs on local machine using sh shell              
          shell: sh                                                             
          before_script: |                                                      
            date                                                                
            echo "these commands will be run"                                   
        csh:                                                                    
          description: submit jobs on local machine using csh shell             
          shell: csh -x                                                         
        tcsh:                                                                   
          description: submit jobs on local machine using tcsh shell            
          shell: /bin/tcsh                                                      
        zsh:                                                                    
          description: submit jobs on local machine using zsh shell             
          shell: /bin/zsh                                                       
                                                                                
    compilers:                                                                  
      find:                                                                     
        gcc: "^(gcc|GCC|PrgEnv-gnu)"                                            
        intel: "^(intel|Intel|PrgEnv-intel)"                                    
        cray: "^(cray|PrgEnv-cray)"                                             
        clang: "^(clang|Clang)"                                                 
        cuda: "^(cuda|CUDA)"                                                    
        pgi: "^(pgi|PGI|PrgEnv-pgi)"                                            
                                                                                
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
          gcc@7.2.0:                                                            
            cc: 'cc'                                                            
            cxx: 'cxx'                                                          
            fc: 'fc'                                                            
            module:                                                             
              load:                                                             
              - gcc/7.2.0                                                       
        intel:                                                                  
          intel@2019:                                                           
            cc: 'icc'                                                           
            cxx: 'icpc'                                                         
            fc: 'ifort'                                                         
            module:                                                             
              purge: True                                                       
              load:                                                             
              - gcc/7.2.0                                                       
              - intel/2019                                                      
        cray:                                                                   
          craype@2.6.2:                                                         
            cc: 'cc'                                                            
            cxx: 'CC'                                                           
            fc: 'fc'                                                            
            module:                                                             
              load: [craype/2.6.2]                                              
              swap: [PrgEnv-gnu, PrgEnv-cray]                                   
                                                                                
        clang:                                                                  
          clang@12.0.0:                                                         
            cc: 'clang'                                                         
            cxx: 'clang++'                                                      
            fc: 'None'                                                          
            module:                                                             
              load: [clang/12.0]                                                
        cuda:                                                                   
          cuda@11.0:                                                            
            cc: 'nvcc'                                                          
            cxx: 'nvcc'                                                         
            fc: 'None'                                                          
            module:                                                             
              load: [cuda/11.0]                                                 
        pgi:                                                                    
          pgi@18.0:                                                             
            cc: 'pgcc'                                                          
            cxx: 'pgc++'                                                        
            fc: 'pgfortran'                                                     
            module:                                                             
              load: [pgi/18.0]                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: none                                                            
    poolsize: 1                                                                 
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
    executors:                                                                  
      local:                                                                    
        bash:                                                                   
          description: submit jobs on local machine                             
          shell: bash -v                                                        
      slurm:                                                                    
        haswell:                                                                
          launcher: sbatch                                                      
          options: ["-p haswell", "-t 00:10"]                                   
      lsf:                                                                      
        batch:                                                                  
          launcher: bsub                                                        
          queue: batch                                                          
          options: ["-q batch", "-t 00:10"]                                     
      cobalt:                                                                   
        normal:                                                                 
          launcher: qsub                                                        
          queue: normal                                                         
          options: ["-n 1", "-t 10"]                                            
      container:                                                                
        ubuntu:                                                                 
          image: ubuntu:20.04                                                   
          platform: docker                                                      
          mounts: "/tmp:/tmp"                                                   
          options: "--rm"                                                       
          description: submit jobs on ubuntu container                          
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
    executors:                                                                  
      defaults:                                                                 
        pollinterval: 10                                                        
        maxpendtime: 45                                                         
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      lsf:                                                                      
        batch:                                                                  
          description: "LSF Executor name 'batch' that submits jobs to 'batch' q
          queue: batch                                                          
          account: developer                                                    
          options: ["-W 20"]                                                    
          before_script: |                                                      
            time                                                                
            echo "commands run before job"                                      
        test:                                                                   
          description: "LSF Executor name 'test' that submits jobs to 'test' que
          launcher: bsub                                                        
          queue: test                                                           
          account: qa                                                           
          options: ["-W 20"]                                                    
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
    testdir: /tmp/buildtest                                                     
    executors:                                                                  
      defaults:                                                                 
        pollinterval: 20                                                        
        maxpendtime: 30                                                         
        account: admin                                                          
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      slurm:                                                                    
        normal:                                                                 
          options: ["-C haswell"]                                               
          qos: normal                                                           
          before_script: |                                                      
            time                                                                
            echo "commands run before job"                                      
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
system:                                                                         
  generic:                                                                      
    hostnames: ['.*']                                                           
    moduletool: lmod                                                            
    poolsize: 1                                                                 
    executors:                                                                  
      defaults:                                                                 
        maxpendtime: 30                                                         
      local:                                                                    
        bash:                                                                   
          description: submit jobs via bash shell                               
          shell: bash                                                           
      cobalt:                                                                   
        knl:                                                                    
          queue: knl                                                            
                                                                                
        haswell:                                                                
          queue: haswell                                                        
    buildspecs:                                                                 
      # whether to rebuild cache file automatically when running `buildtest buil
      rebuild: False                                                            
      # limit number of records to display when running `buildtest buildspec fin
      count: 15                                                                 
      # format fields to display when running `buildtest buildspec find`, By def
      format: "name,description"                                                
      # enable terse mode                                                       
      terse: False                                                              
                                                                                
    report:                                                                     
      count: 25                                                                 
      #enable terse mode for report                                             
      terse: False                                                              
      format: "name,id,state,runtime,returncode"                                
                                                                                
    compilers:                                                                  
      compiler:                                                                 
        gcc:                                                                    
          default:                                                              
            cc: /usr/bin/gcc                                                    
            cxx: /usr/bin/g++                                                   
            fc: /usr/bin/gfortran

Global Schema Examples

buildtest schema -n global.schema.json --example
$ buildtest schema -n global.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  # invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$"
  (badname:                                                                     
    type: script                                                                
    run: "ping login 1"                                                         
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
# this test fails because it exceeds 32 character length for test name          
buildspecs:                                                                     
  _________this_test_exceeds_character_length______________:                    
    type: script                                                                
    run: hostname                                                               
    executor: generic.local.bash                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  hostname:                                                                     
    type: script                                                                
    run: "hostname"                                                             
maintainers: [shahzebsiddiqui, shahzebsiddiqui]                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
# wrong type for maintainers key, expects a string                              
maintainers: 1                                                                  
buildspecs:                                                                     
  hostname:                                                                     
    type: script                                                                
    run: "hostname"                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  # testing all caps                                                            
  ABCDEFGHIJKLMNOPQRSTUVWXYZ:                                                   
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing all lowercase letters                                               
  abcdefghijklmnopqrstuvwxyz:                                                   
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' in beginning followed by all numbers                            
  _0123456789:                                                                  
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' in combination with caps, lowercase and numbers                 
  _ABCDEFabcdef0123456789:                                                      
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '_' at end of word                                                  
  abcdefghijklmnopqrstuvwxyz_:                                                  
    type: script                                                                
    run: "hostname"                                                             
                                                                                
  # testing '.' in beginning of word                                            
  .helloworld:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '.' in middle of word                                               
  hello.world:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '.' at end of word                                                  
  helloworld.:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '-' in middle of word                                               
  hello-world:                                                                  
    type: script                                                                
    run: hostname                                                               
                                                                                
  # testing '-' at end of word                                                  
  helloworld-:                                                                  
    type: script                                                                
    run: hostname

Script Schema Examples

buildtest schema -n script.schema.json --example
$ buildtest schema -n script.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  missing_run_key:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid key name roon, missing run key                         
    cmd: |                                                                      
        systemctl is-active slurmd                                              
        systemctl is-enabled slurmd | grep enabled                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  empty_tags:                                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: tag list can't be empty, requires one item.                    
    tags: []                                                                    
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_env_type:                                                             
    type: script                                                                
    executor: generic.local.bash                                                
    description: env key should be a dictionary                                 
    env:                                                                        
      - FOO=BAR                                                                 
    run: echo $FOO                                                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  non_int_returncodes_list:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: The returncode must be a list of integers and no numbers       
    run: exit 1                                                                 
    status:                                                                     
      returncode: [1, 2.230]                                                    
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_runtime_min:                                                          
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Invalid type for min property in runtime. Must be an integer o
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: "1"                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_skip_value:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid value for skip, must be boolean                        
    skip: 1                                                                     
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_shell_type:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid shell type must be a string                            
    shell: ["/bin/bash"]                                                        
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_returncode_type:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of invalid return code type            
    run: hostname                                                               
    status:                                                                     
      returncode: ["1"]                                                         
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_value_for_mode:                                                       
    type: script                                                                
    executor: generic.local.bash                                                
    description: The status mode must be 'any' or 'all'                         
    run: exit 0                                                                 
    status:                                                                     
      returncode: 0                                                             
      mode: '1'                                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_tags_value:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid tag value must be all string items                     
    tags: ["network", 400]                                                      
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  runtime_min_must_exceed_0:                                                    
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "The runtime must exceed 0"                                    
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: -1                                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_test_name_&!@#$%:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: "invalid test name"                                            
    run: sleep 1                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_regex_stream:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of invalid regex stream                
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: file                                                            
        exp: "world$"                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_property_filename:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: Invalid property name filename under file_regex                
    run: echo "Hello" > hello.txt                                               
    status:                                                                     
      file_regex:                                                               
        - filename: hello.txt                                                   
          exp: '^(Hello)$'                                                      
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  empty_returncode_list:                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: An empty returncode list will cause an error                   
    run: hostname                                                               
    status:                                                                     
      returncode: []                                                            
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_type_shell_shebang:                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid type for shell shebang, must be a string               
    shebang: ["#!/bin/bash"]                                                    
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_summary:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: summary field must be a string                                 
    summary: 1                                                                  
    run: sleep 1                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_slurm_job_state:                                                      
    type: script                                                                
    executor: generic.local.sh                                                  
    description: invalid value for slurm_job_state, should raise error with enum
    sbatch:                                                                     
      - "-n 2"                                                                  
      - "-q normal"                                                             
      - "-t 10"                                                                 
    run: hostname                                                               
    status:                                                                     
      slurm_job_state: "FINISH"                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  missing_regex_exp:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test fails because of missing key 'exp' in regex          
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: stdout                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_description:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description:                                                                
      - "Multi Line description"                                                
      - "is not accepted"                                                       
    run: sleep 1                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_exists_failure:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: this test will fail validation, because item must be a string  
    run: mkdir -p 1                                                             
    status:                                                                     
      exists: [1]                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  non_int_returncodes:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: The returncode must be an int and not a number                 
    run: exit 1                                                                 
    status:                                                                     
      returncode: 1.01                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  executors_invalid_var_type:                                                   
    type: script                                                                
    executor: "generic.local.(bash|sh|zsh)"                                     
    description: Invalid type field for 'vars'                                  
    tags: [tutorials]                                                           
    run: echo $FOO                                                              
    executors:                                                                  
      generic.local.bash:                                                       
        vars: ["FOO=BAR"]                                                       
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_container_platform:                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: invalid container platform                                     
    container:                                                                  
      platform: "containerd"                                                    
      image: hello-world                                                        
    run: echo 'Test Complete!'                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_metrics_additional_property:                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for additional property for metrics property              
    vars:                                                                       
      FOO: BAR                                                                  
    run: echo $FOO                                                              
    metrics:                                                                    
      foo:                                                                      
        variable: FOO                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  regex_additionalProperties_test:                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: Testing for additional properties in regex field               
    run: hostname                                                               
    status:                                                                     
      regex:                                                                    
        stream: stdout                                                          
        exp: "world$"                                                           
        X: 1                                                                    
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  additionalProperties_test:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: additional properties are not allowed so any invalid key/value 
    FOO: BAR                                                                    
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  executors_additionalProperties:                                               
    type: script                                                                
    executor: "generic.local.(bash|sh|zsh)"                                     
    description: Testing for additional properties in 'executors'               
    tags: [tutorials]                                                           
    run: hostname                                                               
    sbatch: ["-N 4"]                                                            
    executors:                                                                  
      generic.local.bash:                                                       
        sbatch: ["-n 4", "-N 1", "-t 30"]                                       
        FOO: BAR                                                                
      generic.local.sh:                                                         
        sbatch: ["-n 8", "-N 1", "-t 60"]                                       
      generic.local.zsh:                                                        
        sbatch: ["-n 16", "-N 2", "-t 120"]                                     
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  testing_additionalProperties_file_regex:                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: Testing for additional properties in file_regex                
    run: |                                                                      
      echo "Hello" > hello.txt                                                  
    status:                                                                     
      file_regex:                                                               
        - file: hello.txt                                                       
          exp: '^(Hello)$'                                                      
          FOO: BAR                                                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  additionalProperties_status:                                                  
    type: script                                                                
    executor: generic.local.bash                                                
    description: test additional properties in status object. This is not allowe
    sbatch: ["-n 2", "-q normal", "-t 10"]                                      
    run: hostname                                                               
    status:                                                                     
      slurm_job_state: "COMPLETED"                                              
      FOO: BAR                                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_additionalProperties_check:                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: Testing for additional properties in file_count                
    run: hostname                                                               
    status:                                                                     
      file_count:                                                               
        - dir: /tmp                                                             
          count: 0                                                              
          FOO: BAR                                                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  missing_regex_in_metrics:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for missing regex in metrics property, which should result
    run: echo "hello"                                                           
    metrics:                                                                    
      foo:                                                                      
        type: str                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  test_regex_and_file_regex_in_metrics:                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for regex and file_regex in metrics. Only one is allowed a
    run: echo "hello" > file.txt                                                
    metrics:                                                                    
      foo:                                                                      
        type: str                                                               
        file_regex:                                                             
          file: file.txt                                                        
          exp: "hello"                                                          
        regex:                                                                  
          stream: stdout                                                        
          exp: "hello"                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_metrics_type:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: metrics property is an object, testing for type                
    vars:                                                                       
      FOO: BAR                                                                  
    run: echo $FOO                                                              
    metrics: FOO                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  metric_type_property_test:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: metrics 'type' must be a 'str', 'float', 'int'                 
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: tuple                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_metrics_name:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: "Metrics name does not follow pattern"                         
    run: echo "hello" > file.txt                                                
    metrics:                                                                    
      (foo-bar:                                                                 
        type: str                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: "BAR"                                                            
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_filetype_invalid_value:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Invalid value for filetype, it must be 'file', 'dir' or 'symlin
    run: mkdir foo                                                              
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: -1                                                             
          filetype: 'filename'                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_property_depth_failure:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: The depth property in file_count must be greater than 0        
    run: hostname                                                               
    status:                                                                     
      file_count:                                                               
        - dir: /tmp                                                             
          count: 0                                                              
          depth: -1                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_property_count_failure:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: The count property in file_count must be greater than 0        
    run: hostname                                                               
    status:                                                                     
      file_count:                                                               
        - dir: /tmp                                                             
          count: -1                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_minimum_check_on_count:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: The count property must be 0 or greater                        
    run: mkdir foo                                                              
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: -1                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_required_property_check:                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: The dir property in file_count is required                     
    run: hostname                                                               
    status:                                                                     
      file_count:                                                               
        - dir: /tmp                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  minimum_check_file_traverse_limit:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: The minimum value for file_traverse_limit is 1                 
    run: mkdir foo                                                              
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: 1                                                              
          file_traverse_limit: 0                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_lt_example:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Run stream test with metrics example using assert less than    
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: float                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
      scale:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Scale:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
      add:                                                                      
        type: float                                                             
        regex:                                                                  
          exp: 'Add:\s+(\S+)\s+.*'                                              
          stream: stdout                                                        
          item: 1                                                               
      triad:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Triad:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
    status:                                                                     
      assert_lt:                                                                
        comparisons:                                                            
        - name: copy                                                            
          ref: 5000                                                             
        - name: scale                                                           
          ref: 5500                                                             
        - name: add                                                             
          ref: 6000                                                             
        - name: triad                                                           
          ref: 6500                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  container_commands_ubuntu:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: run arbitrary linux commands in ubuntu container               
    container:                                                                  
      platform: "docker"                                                        
      image: ubuntu:latest                                                      
      command: bash -c "cat /etc/os-release"                                    
    run: |                                                                      
      ls -l /etc/os-release || true                                             
                                                                                
  container_options:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: run arbitrary linux commands in ubuntu container               
    container:                                                                  
      platform: "docker"                                                        
      image: ubuntu:latest                                                      
      options: --hostname myhostname                                            
      command: hostname                                                         
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  metric_regex_example:                                                         
    executor: generic.local.bash                                                
    type: script                                                                
    description: capture result metric from output                              
    run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"           
    tags: tutorials                                                             
    metrics:                                                                    
      hpcg_rating_stream:                                                       
        type: float                                                             
        regex:                                                                  
          exp: '(\d+\.\d+)$'                                                    
          stream: stdout                                                        
      hpcg_state_stream:                                                        
        type: str                                                               
        regex:                                                                  
          exp: '(VALID)'                                                        
          stream: stdout                                                        
                                                                                
  metric_file_regex:                                                            
    executor: generic.local.bash                                                
    type: script                                                                
    description: capture result metric from file path                           
    run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" > hpcg.txt
    tags: tutorials                                                             
    metrics:                                                                    
      hpcg_rating_file:                                                         
        type: float                                                             
        file_regex:                                                             
          exp: '(\d+\.\d+)$'                                                    
          file: hpcg.txt                                                        
      hpcg_state_file:                                                          
        type: str                                                               
        file_regex:                                                             
          exp: '(VALID)'                                                        
          file: hpcg.txt                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  status_logical_and:                                                           
    type: script                                                                
    executor: 'generic.local.bash'                                              
    description: 'Using logical AND to check status'                            
    run: |                                                                      
      echo "This is a test"                                                     
      exit 1                                                                    
    status:                                                                     
      mode: and                                                                 
      returncode: 1                                                             
      regex:                                                                    
        stream: stdout                                                          
        exp: 'This is a test'                                                   
                                                                                
  status_logical_or:                                                            
    type: script                                                                
    executor: 'generic.local.bash'                                              
    description: 'Using logical OR to check status'                             
    run: |                                                                      
      echo "This is a test"                                                     
      exit 1                                                                    
    status:                                                                     
      mode: or                                                                  
      returncode: 0                                                             
      regex:                                                                    
        stream: stdout                                                          
        exp: 'This is a test'                                                   
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_traverse_limit:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: Use of file_traverse_limit to limit number of files searched in
    run: |                                                                      
      mkdir foo                                                                 
      touch foo/{1..99}.txt                                                     
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: 50                                                             
          file_traverse_limit: 50                                               
        - dir: foo                                                              
          count: 10                                                             
          file_traverse_limit: 20                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  bash_login_shebang:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    shebang: "#!/bin/bash -l"                                                   
    description: customize shebang line with bash login shell                   
    tags: tutorials                                                             
    run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell'   
    status:                                                                     
      regex:                                                                    
        exp: "^Login Shell$"                                                    
        stream: stdout                                                          
                                                                                
  bash_nonlogin_shebang:                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    shebang: "#!/bin/bash"                                                      
    description: customize shebang line with default bash (nonlogin) shell      
    tags: tutorials                                                             
    run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell'   
    status:                                                                     
      regex:                                                                    
        exp: "^Not Login Shell$"                                                
        stream: stdout                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  regex_on_multiple_files:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test regex on multiple files                                   
    run: |                                                                      
      echo "Hello" > hello.txt                                                  
      buildtest --help > buildtest_help.txt                                     
    status:                                                                     
      file_regex:                                                               
        - file: hello.txt                                                       
          exp: '^(Hello)$'                                                      
        - file: buildtest_help.txt                                              
          exp: '^(usage: buildtest)'                                            
                                                                                
  regex_on_directory_not_supported:                                             
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test regex on directory is not supported                       
    run: |                                                                      
      mkdir -p hello                                                            
      echo "Hello" > hello/hello.txt                                            
    status:                                                                     
      file_regex:                                                               
        - file: hello                                                           
          exp: '^(Hello)$'                                                      
                                                                                
  file_expansion_supported:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test regex with variable and shell expansion                   
    run: |                                                                      
      echo "Hello" > $BUILDTEST_ROOT/hello.txt                                  
      echo "Hello" > $HOME/hello.txt                                            
    status:                                                                     
      file_regex:                                                               
        - file: $BUILDTEST_ROOT/hello.txt                                       
          exp: '^(Hello)$'                                                      
        - file: ~/hello.txt                                                     
          exp: '^(Hello)$'                                                      
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_range_ex:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: Example on assert_range                                        
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: float                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
      scale:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Scale:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
      add:                                                                      
        type: float                                                             
        regex:                                                                  
          exp: 'Add:\s+(\S+)\s+.*'                                              
          stream: stdout                                                        
          item: 1                                                               
      triad:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Triad:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
    status:                                                                     
      assert_range:                                                             
        comparisons:                                                            
        - name: copy                                                            
          lower: 5000                                                           
          upper: 20000                                                          
        - name: scale                                                           
          lower: 4500                                                           
          upper: 20000                                                          
        - name: add                                                             
          lower: 4300                                                           
          upper: 20000                                                          
        - name: triad                                                           
          lower: 5600                                                           
          upper: 20000                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  always_pass:                                                                  
    type: script                                                                
    executor: 'generic.local.sh'                                                
    description: This test will always 'PASS'                                   
    run: exit 1                                                                 
    status:                                                                     
      state: PASS                                                               
                                                                                
  always_fail:                                                                  
    type: script                                                                
    executor: 'generic.local.sh'                                                
    description: This test will always 'FAIL'                                   
    run: exit 0                                                                 
    status:                                                                     
      state: FAIL                                                               
                                                                                
  test_fail_returncode_match:                                                   
    type: script                                                                
    executor: 'generic.local.sh'                                                
    description: This test will 'FAIL' even if we have returncode match         
    run: exit 1                                                                 
    status:                                                                     
      state: FAIL                                                               
      returncode: 1                                                             
                                                                                
  test_pass_returncode_mismatch:                                                
    type: script                                                                
    executor: 'generic.local.sh'                                                
    description: This test will 'PASS' even if we have returncode mismatch      
    run: exit 1                                                                 
    status:                                                                     
      state: PASS                                                               
      returncode: 2                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  string_tag:                                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: tags can be a string                                           
    tags: network                                                               
    run: hostname                                                               
                                                                                
  list_of_strings_tags:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: tags can be a list of strings                                  
    tags: [network, ping]                                                       
    run: ping -c 4 www.google.com                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  executors_metrics_declaration:                                                
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring metrics by executors section                         
    tags: [tutorials]                                                           
    run: echo "Hello World"                                                     
    executors:                                                                  
      generic.local.bash:                                                       
        metrics:                                                                
          hello:                                                                
            type: str                                                           
            regex:                                                              
              stream: stdout                                                    
              exp: "(Hello)"                                                    
      generic.local.sh:                                                         
        metrics:                                                                
          world:                                                                
            type: str                                                           
            regex:                                                              
              stream: stdout                                                    
              exp: "(World)"                                                    
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_gt_example:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Run stream test with metrics example using assert greater than.
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: float                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
      scale:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Scale:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
      add:                                                                      
        type: float                                                             
        regex:                                                                  
          exp: 'Add:\s+(\S+)\s+.*'                                              
          stream: stdout                                                        
          item: 1                                                               
      triad:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Triad:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
    status:                                                                     
      assert_gt:                                                                
        comparisons:                                                            
        - name: copy                                                            
          ref: 5000                                                             
        - name: scale                                                           
          ref: 5500                                                             
        - name: add                                                             
          ref: 6000                                                             
        - name: triad                                                           
          ref: 6500                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  _bin_sh_shell:                                                                
    executor: generic.local.sh                                                  
    type: script                                                                
    description: "/bin/sh shell example"                                        
    shell: /bin/sh                                                              
    tags: [tutorials]                                                           
    run: "bzip2 --help"                                                         
                                                                                
  _bin_bash_shell:                                                              
    executor: generic.local.bash                                                
    type: script                                                                
    description: "/bin/bash shell example"                                      
    shell: /bin/bash                                                            
    tags: [tutorials]                                                           
    run: "bzip2 -h"                                                             
                                                                                
  bash_shell:                                                                   
    executor: generic.local.bash                                                
    type: script                                                                
    description: "bash shell example"                                           
    shell: bash                                                                 
    tags: [tutorials]                                                           
    run: "echo $SHELL"                                                          
                                                                                
  sh_shell:                                                                     
    executor: generic.local.sh                                                  
    type: script                                                                
    description: "sh shell example"                                             
    shell: sh                                                                   
    tags: [tutorials]                                                           
    run: "echo $SHELL"                                                          
                                                                                
  shell_options:                                                                
    executor: generic.local.sh                                                  
    type: script                                                                
    description: "shell options"                                                
    shell: "sh -x"                                                              
    tags: [tutorials]                                                           
    run: |                                                                      
      echo $SHELL                                                               
      hostname                                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  sbatch_example:                                                               
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using sbatch directives                
    sbatch:                                                                     
      - "-t 10:00:00"                                                           
      - "-p normal"                                                             
      - "-N 1"                                                                  
      - "-n 8"                                                                  
    run: hostname                                                               
                                                                                
  bsub_example:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using bsub directives                  
    bsub:                                                                       
      - "-W 00:30"                                                              
      - "-N 1"                                                                  
    run: hostname                                                               
                                                                                
  cobalt_example:                                                               
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test runs hostname using cobalt directives                
    cobalt:                                                                     
      - "-t 30"                                                                 
      - "-n 1"                                                                  
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  declare_shell_bin_zsh:                                                        
    executor: generic.local.zsh                                                 
    type: script                                                                
    description: declare shell zsh                                              
    shell: "zsh"                                                                
    run: hostname                                                               
                                                                                
  declare_shell_zsh:                                                            
    executor: generic.local.zsh                                                 
    type: script                                                                
    description: declare shell /bin/zsh                                         
    shell: "/bin/zsh"                                                           
    run: hostname                                                               
                                                                                
  declare_shell_bin_csh:                                                        
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/csh                                         
    shell: "/bin/csh"                                                           
    run: hostname                                                               
                                                                                
  declare_shell_csh:                                                            
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/tcsh                                        
    shell: "csh"                                                                
    run: hostname                                                               
                                                                                
  declare_shell_bin_tcsh:                                                       
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell /bin/tcsh                                        
    shell: "/bin/tcsh"                                                          
    run: hostname                                                               
                                                                                
  declare_shell_tcsh:                                                           
    executor: generic.local.csh                                                 
    type: script                                                                
    description: declare shell tcsh                                             
    shell: "tcsh"                                                               
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  pass_test:                                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will always pass                                     
    status:                                                                     
      state: PASS                                                               
    run: |                                                                      
      echo "This test will pass"                                                
      sleep 2                                                                   
                                                                                
  fail_test:                                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run if test 'pass_test' is in state 'PASS'      
    status:                                                                     
      state: FAIL                                                               
    needs:                                                                      
      - pass_test:                                                              
          state: PASS                                                           
    run: |                                                                      
      echo "This test will fail"                                                
      sleep 2                                                                   
                                                                                
  pass_and_fail_test:                                                           
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run if pass_test is 'PASS' and fail_test is 'FAI
    needs:                                                                      
      - pass_test:                                                              
          state: PASS                                                           
      - fail_test:                                                              
          state: FAIL                                                           
    run: |                                                                      
      sleep 2                                                                   
      exit 1                                                                    
    status:                                                                     
      returncode: 1                                                             
                                                                                
  final_test:                                                                   
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test will run after 'pass_test', 'fail_test', and 'pass_and_fai
    needs:                                                                      
      - pass_test                                                               
      - fail_test                                                               
      - pass_and_fail_test:                                                     
          returncode: 1                                                         
    run: |                                                                      
      sleep 2                                                                   
      echo "Hello world"                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
                                                                                
  exit1_fail:                                                                   
    executor: generic.local.bash                                                
    type: script                                                                
    description: exit 1 by default is FAIL                                      
    tags: [tutorials, fail]                                                     
    run: exit 1                                                                 
                                                                                
  exit1_pass:                                                                   
    executor: generic.local.bash                                                
    type: script                                                                
    description: report exit 1 as PASS                                          
    run: exit 1                                                                 
    tags: [tutorials, pass]                                                     
    status:                                                                     
      returncode: [1]                                                           
                                                                                
  returncode_list_mismatch:                                                     
    executor: generic.local.bash                                                
    type: script                                                                
    description: exit 2 failed since it failed to match returncode 1            
    run: exit 2                                                                 
    tags: [tutorials, fail]                                                     
    status:                                                                     
      returncode: [1, 3]                                                        
                                                                                
  returncode_int_match:                                                         
    executor: generic.local.bash                                                
    type: script                                                                
    description: exit 128 matches returncode 128                                
    run: exit 128                                                               
    tags: [tutorials, pass]                                                     
    status:                                                                     
      returncode: 128                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  skip:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test is skipped                                           
    skip: Yes                                                                   
    tags: [tutorials]                                                           
    run: hostname                                                               
                                                                                
  unskipped:                                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test is not skipped                                       
    skip: No                                                                    
    tags: [tutorials]                                                           
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  timelimit_min_max:                                                            
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if its within 1.0-
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: 1.0                                                                
        max: 3.0                                                                
                                                                                
  timelimit_min:                                                                
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if its exceeds min
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: 1.0                                                                
                                                                                
  timelimit_max:                                                                
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "Run a sleep job for 2 seconds and test pass if it's within max
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        max: 5.0                                                                
                                                                                
  timelimit_min_fail:                                                           
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "This test fails because it runs less than mintime of 10 second
    tags: ["tutorials"]                                                         
    run: sleep 2                                                                
    status:                                                                     
      runtime:                                                                  
        min: 10.0                                                               
                                                                                
  timelimit_max_fail:                                                           
    type: script                                                                
    executor: generic.local.sh                                                  
    description: "This test fails because it exceeds maxtime of 1.0 second"     
    tags: ["tutorials"]                                                         
    run: sleep 3                                                                
    status:                                                                     
      runtime:                                                                  
        max: 1.0                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_eq_example:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for assert equality                                       
    vars:                                                                       
      X: 1                                                                      
      Y: 1.5                                                                    
      first: John                                                               
      last: Smith                                                               
    run: |                                                                      
      echo "X: $X"                                                              
      echo "Y: $Y"                                                              
      echo "Name: $first $last"                                                 
    metrics:                                                                    
      x:                                                                        
        type: int                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'X:\s+(\S+)\s+.*'                                                
          item: 1                                                               
      y:                                                                        
        type: float                                                             
        regex:                                                                  
          stream: stdout                                                        
          exp: 'Y:\s+(\S+)\s+.*'                                                
          item: 1                                                               
      first:                                                                    
        type: str                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'Name:\s+(\S+)\s+.*'                                             
          item: 1                                                               
      last:                                                                     
        type: str                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: '(Smith)$'                                                       
          item: 1                                                               
    status:                                                                     
      assert_eq:                                                                
        comparisons:                                                            
        - name: x                                                               
          ref: 1                                                                
        - name: y                                                               
          ref: 1.5                                                              
        - name: first                                                           
          ref: John                                                             
        - name: last                                                            
          ref: Smith                                                            
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  status_exists:                                                                
    type: script                                                                
    executor: generic.local.bash                                                
    description: status check based for file and directory                      
    run: |                                                                      
      mkdir -p $HOME/dirA                                                       
      mkdir -p /tmp/ABC                                                         
      touch file1                                                               
    status:                                                                     
      exists:                                                                   
        - $HOME/dirA                                                            
        - ~/.bashrc                                                             
        - /tmp/ABC                                                              
        - file1                                                                 
  status_exists_failure:                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: status check failure for existence                             
    run: touch foo                                                              
    status:                                                                     
      exists:                                                                   
        - bar                                                                   
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  multiline_run:                                                                
    executor: generic.local.bash                                                
    type: script                                                                
    description: multiline run command                                          
    run: |                                                                      
      echo "1"                                                                  
      echo "2"                                                                  
                                                                                
  single_command_run:                                                           
    executor: generic.local.bash                                                
    type: script                                                                
    description: single command as a string for run command                     
    run: "hostname"                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_ne_example:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Test for assert not equal                                      
    vars:                                                                       
      X: 1                                                                      
      Y: 1.5                                                                    
      first: John                                                               
      last: Smith                                                               
    run: |                                                                      
      echo "X: $X"                                                              
      echo "Y: $Y"                                                              
      echo "Name: $first $last"                                                 
    metrics:                                                                    
      x:                                                                        
        type: int                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'X:\s+(\S+)\s+.*'                                                
          item: 1                                                               
      y:                                                                        
        type: float                                                             
        regex:                                                                  
          stream: stdout                                                        
          exp: 'Y:\s+(\S+)\s+.*'                                                
          item: 1                                                               
      first:                                                                    
        type: str                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'Name:\s+(\S+)\s+.*'                                             
          item: 1                                                               
      last:                                                                     
        type: str                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: '(Smith)$'                                                       
          item: 1                                                               
    status:                                                                     
      assert_ne:                                                                
        comparisons:                                                            
        - name: x                                                               
          ref: 2                                                                
        - name: y                                                               
          ref: 2.5                                                              
        - name: first                                                           
          ref: Robert                                                           
        - name: last                                                            
          ref: Brown                                                            
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  circle_area:                                                                  
    executor: generic.local.bash                                                
    type: script                                                                
    shell: python                                                               
    description: "Calculate circle of area given a radius"                      
    tags: [tutorials, python]                                                   
    run: |                                                                      
      import math                                                               
      radius = 2                                                                
      area = math.pi * radius * radius                                          
      print("Circle Radius ", radius)                                           
      print("Area of circle ", area)                                            
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  executors_vars_env_declaration:                                               
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring env and vars by executors section                    
    tags: [tutorials]                                                           
    run: |                                                                      
      echo "X:" $X                                                              
      echo "Y:" $Y                                                              
      echo $SHELL                                                               
    executors:                                                                  
      generic.local.bash:                                                       
        vars:                                                                   
          X: 1                                                                  
          Y: 3                                                                  
        env:                                                                    
          SHELL: bash                                                           
      generic.local.sh:                                                         
        vars:                                                                   
          X: 2                                                                  
          Y: 4                                                                  
        env:                                                                    
          SHELL: sh                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  runtime_test:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will sleep 5 second but will fail due to runtime 2sec
    status:                                                                     
      runtime:                                                                  
        min: 2.0                                                                
    run: sleep 5                                                                
                                                                                
  runtime_test_pass:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run when runtime_test_pass is PASS              
    needs:                                                                      
      - runtime_test:                                                           
          state: PASS                                                           
    run: echo "Performing some action when test PASS"                           
                                                                                
  runtime_test_fail:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run when runtime_test_pass is FAIL              
    needs:                                                                      
      - runtime_test:                                                           
          state: FAIL                                                           
    run: echo "Performing some action when test FAIL"                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  declare_vars:                                                                 
    executor: generic.local.bash                                                
    type: script                                                                
    description: declaring variables                                            
    vars:                                                                       
      First: Bob                                                                
      Last: Bill                                                                
    run: |                                                                      
      echo "First:" $First                                                      
      echo "Last:" $Last                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  status_returncode_by_executors:                                               
    type: script                                                                
    executor: "generic.local.(bash|sh)"                                         
    description: define status per executor type.                               
    tags: [tutorials]                                                           
    run: exit 0                                                                 
    executors:                                                                  
      generic.local.bash:                                                       
        status:                                                                 
          returncode: [0, 2]                                                    
      generic.local.sh:                                                         
        status:                                                                 
          returncode: 1                                                         
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  bash_env_variables:                                                           
    executor: generic.local.bash                                                
    description: Declare environment variables in default shell (bash)          
    type: script                                                                
    env:                                                                        
      FIRST_NAME: avocado                                                       
      LAST_NAME: dinosaur                                                       
    tags: [tutorials]                                                           
    run: |                                                                      
      hostname                                                                  
      whoami                                                                    
      echo $USER                                                                
      printf "${FIRST_NAME} ${LAST_NAME}\n"                                     
                                                                                
  csh_env_declaration:                                                          
    executor: generic.local.csh                                                 
    type: script                                                                
    description: "csh shell example to declare environment variables"           
    shell: /bin/csh                                                             
    tags: [tutorials]                                                           
    env:                                                                        
      SHELL_NAME: "csh"                                                         
    run: echo "This is running $SHELL_NAME"                                     
                                                                                
  tcsh_env_declaration:                                                         
    executor: generic.local.csh                                                 
    type: script                                                                
    description: "tcsh shell example to declare environment variables"          
    shell: /bin/tcsh                                                            
    tags: [tutorials]                                                           
    env:                                                                        
      path: "/usr/local/bin:$PATH"                                              
    run: echo $path                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  symlink_test:                                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: status check based on symbolic link                            
    run: |                                                                      
      ln -s /tmp scratch                                                        
      ln -s $HOME/.bashrc $HOME/.bashrc_link                                    
    status:                                                                     
      is_symlink:                                                               
        - scratch                                                               
        - $HOME/.bashrc_link                                                    
        - ~/.bashrc_link                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  hello_world_container_no_commands:                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: run container with no commands                                 
    container:                                                                  
      platform: "docker"                                                        
      image: hello-world                                                        
    run: echo 'Test Complete!'                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  executors_sbatch_declaration:                                                 
    type: script                                                                
    executor: 'generic.local.(bash|sh)'                                         
    description: Declaring env and vars by executors section                    
    tags: [tutorials]                                                           
    run: hostname                                                               
    sbatch: ["-N 4"]                                                            
    executors:                                                                  
      generic.local.bash:                                                       
        sbatch: ["-n 4", "-N 1", "-t 30"]                                       
      generic.local.sh:                                                         
        sbatch: ["-n 8", "-N 1", "-t 60"]                                       
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_by_expression:                                                     
    type: script                                                                
    executor: generic.local.bash                                                
    description: file count by expression                                       
    run: |                                                                      
      mkdir -p /tmp                                                             
      touch /tmp/foo{1..5}.txt                                                  
      ls -l /tmp/foo*.txt                                                       
      filenames=$(find $BUILDTEST_ROOT/buildtest -type f  \( -name "defaults.py"
      totalfiles=$(find $BUILDTEST_ROOT/buildtest -type f  \( -name "defaults.py
      echo "Filenames: $filenames"                                              
      echo "Total files: $totalfiles"                                           
    status:                                                                     
      file_count:                                                               
        - dir: /tmp                                                             
          filepattern: 'foo[1-5].txt$'                                          
          count: 5                                                              
        - dir: '$BUILDTEST_ROOT/buildtest'                                      
          filepattern: '(defaults|main).py$'                                    
          count: 2                                                              
          depth: 1                                                              
                                                                                
  file_extension_and_filepattern:                                               
    type: script                                                                
    executor: generic.local.bash                                                
    description: file count by file extension and file pattern                  
    run: |                                                                      
      touch foo{1..5}.txt                                                       
      touch {conf,main}.py                                                      
      ls -l                                                                     
    status:                                                                     
      file_count:                                                               
        - dir: .                                                                
          ext: '.txt'                                                           
          filepattern: '(conf|main).py$'                                        
          count: 7                                                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  jobA:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: no job dependency                                              
    run: |                                                                      
      echo "This job has no dependency"                                         
      sleep 5                                                                   
                                                                                
  jobB:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: job dependency on jobA                                         
    needs: [jobA]                                                               
    run: |                                                                      
      echo "This job depends on jobA"                                           
      sleep 2                                                                   
                                                                                
  jobC:                                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: job dependency on jobA and jobB                                
    needs: [jobA, jobB]                                                         
    run: |                                                                      
      echo "This job depends on jobA and jobB"                                  
      sleep 2                                                                   
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_and_dir_checks:                                                          
    type: script                                                                
    executor: generic.local.bash                                                
    description: status check for files and directories                         
    run: hostname                                                               
    status:                                                                     
      is_dir:                                                                   
        - $HOME                                                                 
        - $HOME/.bashrc                                                         
        - /tmp                                                                  
  combined_file_and_dir_checks:                                                 
    type: script                                                                
    executor: generic.local.bash                                                
    description: status check for files and directories                         
    run: hostname                                                               
    status:                                                                     
      is_dir:                                                                   
        - $HOME                                                                 
        - /tmp                                                                  
      is_file:                                                                  
        - $HOME/.bashrc                                                         
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  bind_mount_in_container:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: run a python script in container                               
    container:                                                                  
      platform: "docker"                                                        
      image: ubuntu:latest                                                      
      mounts: "/tmp:/tmp"                                                       
      command: bash -c "echo 'hello world' > /tmp/hello.txt"                    
    run: |                                                                      
      cat /tmp/hello.txt                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  stream_test:                                                                  
    type: script                                                                
    executor: generic.local.bash                                                
    description: Run stream test with metrics example using assert greater equal
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: float                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
      scale:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Scale:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
      add:                                                                      
        type: float                                                             
        regex:                                                                  
          exp: 'Add:\s+(\S+)\s+.*'                                              
          stream: stdout                                                        
          item: 1                                                               
      triad:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Triad:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
    status:                                                                     
      assert_ge:                                                                
        mode: and                                                               
        comparisons:                                                            
        - name: copy                                                            
          ref: 5000                                                             
        - name: scale                                                           
          ref: 5500                                                             
        - name: add                                                             
          ref: 6000                                                             
        - name: triad                                                           
          ref: 6500                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  assert_le_example:                                                            
    type: script                                                                
    executor: generic.local.bash                                                
    description: Run stream test with metrics example using assert less than equ
    env:                                                                        
      OMP_NUM_THREADS: 4                                                        
    run: |                                                                      
      wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 
      gcc -openmp -o stream stream.c                                            
      ./stream                                                                  
    metrics:                                                                    
      copy:                                                                     
        type: float                                                             
        regex:                                                                  
          exp: 'Copy:\s+(\S+)\s+.*'                                             
          stream: stdout                                                        
          item: 1                                                               
      scale:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Scale:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
      add:                                                                      
        type: float                                                             
        regex:                                                                  
          exp: 'Add:\s+(\S+)\s+.*'                                              
          stream: stdout                                                        
          item: 1                                                               
      triad:                                                                    
        type: float                                                             
        regex:                                                                  
          exp: 'Triad:\s+(\S+)\s+.*'                                            
          stream: stdout                                                        
          item: 1                                                               
    status:                                                                     
      assert_le:                                                                
        comparisons:                                                            
        - name: copy                                                            
          ref: 5000                                                             
        - name: scale                                                           
          ref: 5500                                                             
        - name: add                                                             
          ref: 6000                                                             
        - name: triad                                                           
          ref: 6500                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  test1:                                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will pass with exit 1                                
    run: exit 1                                                                 
    status:                                                                     
      state: PASS                                                               
                                                                                
  test2:                                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run if test1 has returncode 1                   
    run: exit 2                                                                 
    status:                                                                     
      state: PASS                                                               
    needs:                                                                      
      - test1:                                                                  
          returncode: 1                                                         
                                                                                
  test3:                                                                        
    type: script                                                                
    executor: generic.local.bash                                                
    description: This test will run if test1 has returncode 1 and test2 has retu
    run: exit 1                                                                 
    status:                                                                     
      state: PASS                                                               
    needs:                                                                      
      - test1:                                                                  
          returncode: 1                                                         
      - test2:                                                                  
          returncode: 2                                                         
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  contains_and_not_contains:                                                    
    type: script                                                                
    executor: generic.local.bash                                                
    description: Status check based on contains and not contains where test pass
    vars:                                                                       
      X: 1                                                                      
    run: |                                                                      
      echo "X: $X"                                                              
    metrics:                                                                    
      x:                                                                        
        type: int                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'X:\s+(\S+)\s+.*'                                                
          item: 1                                                               
    status:                                                                     
      contains:                                                                 
        comparisons:                                                            
        - name: x                                                               
          ref: [1, 2, 4, 8]                                                     
      not_contains:                                                             
        comparisons:                                                            
        - name: x                                                               
          ref: [2, 4]                                                           
  assert_contains_fail:                                                         
    type: script                                                                
    executor: generic.local.bash                                                
    description: Status check based on contains where test fails                
    vars:                                                                       
      X: 1                                                                      
    run: |                                                                      
      echo "X: $X"                                                              
    metrics:                                                                    
      x:                                                                        
        type: int                                                               
        regex:                                                                  
          stream: stdout                                                        
          exp: 'X:\s+(\S+)\s+.*'                                                
          item: 1                                                               
    status:                                                                     
      contains:                                                                 
        comparisons:                                                            
        - name: x                                                               
          ref: ['1', 2, 4, 8]                                                   
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  summary_example:                                                              
    type: script                                                                
    executor: generic.local.bash                                                
    description: The summary field can be a multi-line string and exceed 80 char
    tags: [tutorials]                                                           
    summary: |                                                                  
      This is a long description of test that                                   
      can exceed 80 characters and be multiline                                 
    run: hostname                                                               
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_on_directory:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: file count check in directory                                  
    run: |                                                                      
      mkdir -p foo                                                              
      touch foo/{1..5}                                                          
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: 5                                                              
  file_count_by_extension:                                                      
    type: script                                                                
    executor: generic.local.bash                                                
    description: file count by extension                                        
    run: |                                                                      
      mkdir -p foo/bar                                                          
      touch foo/{1..5}.sh                                                       
      touch foo/bar/{1..3}.py foo/bar/{1..3}.txt                                
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          ext: '.sh'                                                            
          depth: 1                                                              
          count: 5                                                              
        - dir: foo/bar                                                          
          ext: ['.py', '.txt']                                                  
          count: 6                                                              
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  file_count_by_filetype:                                                       
    type: script                                                                
    executor: generic.local.bash                                                
    description: Count the number of directories and symbolic links             
    run: |                                                                      
      mkdir -p foo/{bar,baz}                                                    
      find foo -type dir                                                        
      ln -s foo/bar foo/bar.link                                                
      ln -s foo/baz foo/baz.link                                                
    status:                                                                     
      file_count:                                                               
        - dir: foo                                                              
          count: 3                                                              
          filetype: 'dir'                                                       
        - dir: foo                                                              
          count: 2                                                              
          filetype: 'symlink'

Spack Schema Examples

buildtest schema -n spack.schema.json --example
$ buildtest schema -n spack.schema.json --example
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  specs_must_be_list_of_strings:                                                
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: 'specs must be a list of strings'                              
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs: [1, zlib]                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_load_invalid_example:                                                   
    type: spack                                                                 
    executor: generic.local.bash                                                
    description: "Run spack load for m4 package and run tests"                  
    tags: [spack]                                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      load:                                                                     
        options: '--only package'                                               
        specs: [1.5]                                                            
      test:                                                                     
        run:                                                                    
          specs: ['m4']                                                         
        results:                                                                
          option: "-l"                                                          
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_env_deactivate_first:                                                   
    type: spack                                                                 
    executor: generic.local.bash                                                
    description: "deactivate a spack environment first prior to activating it"  
    tags: [spack]                                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          dir: $HOME/spack-envs/m4                                              
        deactivate: 2.0                                                         
        activate:                                                               
          dir: $HOME/spack-envs/m4                                              
        specs:                                                                  
          - 'm4'                                                                
      install:                                                                  
        option: ''                                                              
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $HOME/spack-envs/m4                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_test_run_invalid_spec:                                                  
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "specs property requires a list of strings. "                  
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib']                                                   
      test:                                                                     
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 1]                                                      
        results:                                                                
          option: '-f'                                                          
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $SPACK_ROOT                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  invalid_type_mirror_field:                                                    
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: The mirror field must be a key value pair                      
    spack:                                                                      
      root: $HOME/spack                                                         
      mirror: https://caches.e4s.io                                             
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - zlib                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_test_additionalProperties:                                              
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: "Check for additionalProperties in test section. FOO key is not
    tags: [spack]                                                               
    pre_cmds: |                                                                 
      cd /tmp                                                                   
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: /tmp/spack                                                          
      verify_spack: false                                                       
      install:                                                                  
        specs: ['m4', 'zlib']                                                   
      test:                                                                     
        FOO: BAR                                                                
        remove_tests: true                                                      
        run:                                                                    
          specs: ['m4', 'zlib']                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  additionalProperties_spack_field:                                             
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: additional Properties can't be specified in spack section      
    spack:                                                                      
      root: $HOME/spack                                                         
      FOO: BAR                                                                  
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - zlib                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_create_from_manifest:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack enviromment from manifest file                    
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_mirror:                                                                   
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in e
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        mirror:                                                                 
          h5: /path/to/mirror                                                   
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  install_without_env:                                                          
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: install specs without environment                              
    spack:                                                                      
      root: $HOME/spack/                                                        
      install:                                                                  
        options: '--cache-only'                                                 
        specs: ['m4', 'bzip2']                                                  
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  pre_post_cmd_spack_install:                                                   
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: run commands before and after spack using pre_cmds and post_cmd
    pre_cmds: |                                                                 
      cd $HOME                                                                  
      git clone https://github.com/spack/spack                                  
    spack:                                                                      
      root: $HOME/spack/                                                        
      install:                                                                  
        options: '--cache-only'                                                 
        specs: ['m4', 'bzip2']                                                  
    post_cmds: |                                                                
      spack find                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_create_directory:                                                         
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack environment by directory                          
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          dir: $HOME/spack-env/myproject                                        
        specs:                                                                  
          - 'm4'                                                                
          - 'zlib@1.2.11'                                                       
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_concretized_install:                                                      
    type: spack                                                                 
    description: run 'spack concretize -f' in an environment and install specs  
    executor: generic.local.sh                                                  
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
          manifest: $HOME/spack.yaml                                            
        concretize: true                                                        
      install:                                                                  
        options: '--cache-only'                                                 
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_activate:                                                                 
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Activate spack environment by name                             
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        activate:                                                               
          name: myproject                                                       
        specs:                                                                  
          - m4                                                                  
          - zlib                                                                
      install:                                                                  
        options: ''                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  vars_multi_executors:                                                         
    type: spack                                                                 
    executor: "generic.local.(sh|bash)"                                         
    description: "variable declaration with multiple executors"                 
    tags: [spack]                                                               
    executors:                                                                  
      generic.local.sh:                                                         
        vars:                                                                   
          FOO: BAR                                                              
      generic.local.bash:                                                       
        vars:                                                                   
          HELLO: WORLD                                                          
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        specs:                                                                  
          - 'm4'                                                                
        activate:                                                               
          name: m4                                                              
        concretize: true                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  sbatch_field_with_spack:                                                      
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Specify sbatch field with spack schema                         
    sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"]                             
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  env_create_name:                                                              
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Create spack environment by name                               
    spack:                                                                      
      root: $HOME/spack/                                                        
      env:                                                                      
        create:                                                                 
          name: myproject                                                       
        specs:                                                                  
          - m4                                                                  
          - zlib                                                                
      install:                                                                  
        options: ''                                                             
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_env_deactivate_first:                                                   
    type: spack                                                                 
    executor: generic.local.bash                                                
    description: "deactivate a spack environment first prior to activating it"  
    tags: [spack]                                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          dir: $HOME/spack-envs/m4                                              
        deactivate: true                                                        
        activate:                                                               
          dir: $HOME/spack-envs/m4                                              
        specs:                                                                  
          - 'm4'                                                                
      install:                                                                  
        option: ''                                                              
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $HOME/spack-envs/m4                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_sbatch_multi_executors:                                                 
    type: spack                                                                 
    executor: "generic.local.(sh|bash)"                                         
    description: "sbatch directives can be defined in spack schema"             
    tags: [spack]                                                               
    executors:                                                                  
      generic.local.sh:                                                         
        sbatch: ["-N 1", "-t 20"]                                               
      generic.local.bash:                                                       
        sbatch: ["-N 8", "-t 10"]                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        specs:                                                                  
          - 'm4'                                                                
        activate:                                                               
          name: m4                                                              
        concretize: true                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  skip_test_in_spack:                                                           
    skip: True                                                                  
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: This test will be skipped                                      
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  var_declaration_in_spack:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: Define variables and environment variables in spack            
    vars:                                                                       
      FOO: "BAR"                                                                
    env:                                                                        
      SPACK_GNUPGHOME: "$HOME/.gnupg"                                           
    spack:                                                                      
      root: $HOME/spack                                                         
      install:                                                                  
        specs: ['m4']                                                           
        options: "-v"                                                           
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  remove_spack_environment_automatically:                                       
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: remove spack environment automatically                         
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        create:                                                                 
          remove_environment: true                                              
          name: m4                                                              
        activate:                                                               
          name: m4                                                              
        specs:                                                                  
          - 'm4'                                                                
                                                                                
    post_cmds: |                                                                
      spack find                                                                
      rm -rf $SPACK_ROOT                                                        
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  remove_spack_environment:                                                     
    type: spack                                                                 
    executor: generic.local.sh                                                  
    description: remove spack environment explicitly before creating environment
    spack:                                                                      
      root: $HOME/spack                                                         
      env:                                                                      
        rm:                                                                     
          name: m4                                                              
        create:                                                                 
          name: m4                                                              
        activate:                                                               
          name: m4                                                              
        specs:                                                                  
          - 'm4'                                                                
                                                                                
─ /home/docs/checkouts/readthedocs.org/user_builds/buildtest/checkouts/stable… ─
buildspecs:                                                                     
  spack_load_valid_example:                                                     
    type: spack                                                                 
    executor: generic.local.bash                                                
    description: "Run spack load for m4 package and run tests"                  
    tags: [spack]                                                               
    spack:                                                                      
      root: $HOME/spack                                                         
      load:                                                                     
        options: '--only package'                                               
        specs: [m4]                                                             
      test:                                                                     
        run:                                                                    
          specs: ['m4']                                                         
        results:                                                                
          option: "-l"