#!/bin/bash
# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# NAME
#     test_header
#
# SYNOPSIS
#     . $CYLC_DIR/t/lib/bash/test_header
#
# DESCRIPTION
#     Interface for constructing tests under a TAP harness (the "prove"
#     command).
#
# FUNCTIONS
#     set_test_number N
#         echo a total number of tests for TAP to read.
#     ok TEST_NAME
#         echo a TAP OK message for TEST_NAME.
#     fail TEST_NAME
#         echo a TAP fail message for TEST_NAME. If $CYLC_TEST_DEBUG is set,
#         cat $TEST_NAME.stderr to stderr.
#     run_ok TEST_NAME COMMAND ...
#         Run COMMAND with any following options/arguments and store stdout
#         and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
#         This is expected to have a return code of 0, which ok's the test.
#     run_fail TEST_NAME COMMAND ...
#         Run COMMAND with any following options/arguments and store stdout
#         and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
#         This is expected to have a non-zero return code, which ok's the test.
#     cmp_ok FILE_TEST [FILE_CONTROL]
#         Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin
#         if FILE_CONTROL is "-" or missing). By default, it uses "diff -u" to
#         compare files. However, if an alternate command such as "xxdiff -D"
#         is desirable (e.g. for debugging),
#         "export CYLC_TEST_DIFF_CMD=xxdiff -D".
#     cmp_json_ok FILE_TEST [FILE_CONTROL]
#         Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin
#         if FILE_CONTROL is "-" or missing), where FILE_TEST must also be in
#         valid JSON format.
#     contains_ok FILE_TEST [FILE_CONTROL]
#         Make sure that each line in FILE_TEST is present in FILE_CONTROL
#         (stdin if FILE_CONTROL is "-" or missing).
#     grep_ok PATTERN FILE
#         Run "grep -q -e PATTERN FILE".
#     grep_fail PATTERN FILE
#         Run "grep -q -e PATTERN FILE", expect no match.
#     count_ok PATTERN FILE COUNT
#         Test that PATTERN occurs in exactly COUNT lines of FILE.
#     exists_ok FILE
#         Test that FILE exists
#     exists_fail FILE
#         Test that FILE does not exist
#     init_suite TEST_NAME [SOURCE]
#         Create a suite from SOURCE's "suite.rc" called:
#         "cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}"
#         Provides SUITE_NAME and SUITE_RUN_DIR variables.
#     install_suite TEST_NAME SOURCE
#         Same as init_suite, but SOURCE must be a directory containing a
#         "suite.rc" file. Also provides SOURCE as ORIG_SUITE_NAME variable.
#     log_scan TEST_NAME FILE ATTEMPTS DELAY PATTERN...
#         Monitor FILE polling every DELAY seconds for maximum of ATTEMPTS
#         tries grepping for each PATTERN in turn. Tests will only pass if the
#         PATTERNs appear in FILE in the correct order. Runs one test per
#         pattern, each prefixed by TEST_NAME.
#     mock_smtpd_init
#         Start a mock SMTP server daemon for testing. Write host:port setting
#         to the variable TEST_SMTPD_HOST. Write pid of daemon to
#         TEST_SMTPD_PID. Write log to TEST_SMTPD_LOG.
#     mock_smtpd_kill
#         Kill the mock SMTP server daemon process.
#     purge_suite SUITE_NAME
#         Tidy up test directories for SUITE_NAME.
#     poll COMMAND
#         Run COMMAND in 1 second intervals for a minute until COMMAND returns
#         a non-zero value.
#     skip N SKIP_REASON
#         echo "ok $((++T)) # skip SKIP_REASON" N times.
#     skip_all SKIP_REASON
#         echo "1..0 # SKIP SKIP_REASON" and exit.
#     ssh_install_cylc HOST
#         Install cylc on a remote host.
#     create_test_globalrc [PRE [POST]]
#         Create a new global config file $PWD/etc from site global-tests.rc
#         with PRE and POST pre- and ap-pended (PRE for top level items with no
#         section heading).
#     set_test_remote_host
#         set CYLC_TEST_HOST from global config, for remote job tests.
#         (Remote job tests should really use set_test_remote, below, however).
#     set_test_remote
#         set CYLC_TEST_HOST and CYLC_TEST_OWNER for remote job tests.
#     port_is_busy $PORT
#         Return 0 if $PORT is busy or 1 if $PORT is not busy.
#     cylc_ws_init $NS $UTIL
#         Start a cylc web service server. Test server OK. Write to these shell
#         variables on success:
#         * TEST_CYLC_WS_PID - PID of the service server
#         * TEST_CYLC_WS_PORT - Port of the service server on localhost.
#         * TEST_CYLC_WS_URL - URL of service server.
#         E.g. cylc_ws_init cylc review
#     cylc_ws_kill
#         Kill a web service server started by "cylc_ws_init" and remove
#         generated log and status files.
#     cylc_ws_json_greps TEST_KEY JSON-FILE EXPECTED [EXPECTED ...]
#         Load JSON content in argument 1
#         Each remaining argument is an expected content in the JSON data in
#         the form: [keys, value] where "keys" is a list containing keys or
#         indexes to an expected item in the data structure, and the value is
#         an expected value. A key in keys can be a simple dict key or an
#         array index. It can also be a dict {attr_key: attr_value, ...}. In
#         which case, the expected data item is under a list of dicts, where a
#         unique dict in the list contains all elements attr_key: attr_value.
#     set_test_remote_host
#         set CYLC_TEST_HOST from global config, for remote job tests.
#         (Remote job tests should really use set_test_remote, below, however).
#     set_test_remote
#         set CYLC_TEST_HOST and CYLC_TEST_OWNER for remote job tests.
#-------------------------------------------------------------------------------
set -eu

FAILURES=0
SIGNALS="EXIT INT"
TEST_DIR=
TEST_RHOST_CYLC_DIR=
FINALLY() {
    for S in ${SIGNALS}; do
        trap '' "${S}"
    done
    if [[ -n "${TEST_DIR}" ]]; then
        cd ~
        rm -rf "${TEST_DIR}"
    fi
    if [[ -n "${TEST_RHOST_CYLC_DIR}" ]]; then
        ssh -oBatchMode=yes -oConnectTimeout=5 "${TEST_RHOST_CYLC_DIR%%:*}" \
            "rm -fr ${TEST_RHOST_CYLC_DIR#*:}"
    fi
    if [[ -n "${TEST_SMTPD_PID:-}" ]]; then
        kill "${TEST_SMTPD_PID}"
    fi
    if (($FAILURES > 0)); then
        echo -e "\n    stdout and stderr stored in: ${TEST_LOG_DIR}" >&2
        if "${SUITE_RUN_FAILS}"; then
            echo -e "    suite logs can be found under: ${SUITE_LOG_DIR}" >&2
        fi
    fi
}
for S in ${SIGNALS}; do
    trap "FINALLY ${S}" "${S}"
done

TEST_NUMBER=0

set_test_number() {
    echo "1..$1"
}

ok() {
    echo "ok $((++TEST_NUMBER)) - $@"
}

fail() {
    ((++FAILURES))
    echo "not ok $((++TEST_NUMBER)) - $@"
    if [[ -n "${CYLC_TEST_DEBUG:-}" ]]; then
        echo >'/dev/tty'
        echo "${TEST_NAME_BASE} ${TEST_NAME}" >'/dev/tty'
        cat "${TEST_NAME}.stderr" >'/dev/tty'
    fi
}

run_ok() {
    local TEST_NAME="$1"
    shift 1
    if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
        fail "${TEST_NAME}"
        mkdir -p "${TEST_LOG_DIR}"
        cp -p "${TEST_NAME}.stdout" "${TEST_LOG_DIR}/${TEST_NAME}.stdout"
        cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
        return
    fi
    ok "${TEST_NAME}"
}

run_fail() {
    local TEST_NAME="$1"
    shift 1
    if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
        fail "${TEST_NAME}"
        mkdir -p "${TEST_LOG_DIR}"
        cp -p "${TEST_NAME}.stdout" "${TEST_LOG_DIR}/${TEST_NAME}.stdout"
        cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
        return
    fi
    ok "${TEST_NAME}"
}

suite_run_ok() {
    local TEST_NAME="$1"
    shift 1
    if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
        ok "${TEST_NAME}"
        return
    fi
    fail "${TEST_NAME}"
    SUITE_RUN_FAILS=true
    SUITE_LOG_DIR="${SUITE_RUN_DIR}/log/suite"
    mkdir -p "${SUITE_LOG_DIR}" # directory may not exist if run fails very early
    cp -p "${TEST_NAME}.stdout" "${SUITE_LOG_DIR}/out"
    cp -p "${TEST_NAME}.stderr" "${SUITE_LOG_DIR}/err"
}

suite_run_fail() {
    local TEST_NAME="$1"
    shift 1
    if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
        ok "${TEST_NAME}"
        return
    fi
    fail "${TEST_NAME}"
    SUITE_RUN_FAILS=true
    SUITE_LOG_DIR="${SUITE_RUN_DIR}/log/suite"
    mkdir -p "${SUITE_LOG_DIR}" # directory may not exist if run fails very early
    cp -p "${TEST_NAME}.stdout" "${SUITE_LOG_DIR}/out"
    cp -p "${TEST_NAME}.stderr" "${SUITE_LOG_DIR}/err"
}

cmp_ok() {
    local FILE_TEST="$1"
    local FILE_CONTROL="${2:--}"
    local TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok"
    local DIFF_CMD=${CYLC_TEST_DIFF_CMD:-'diff -u'}
    if ${DIFF_CMD} "${FILE_CONTROL}" "${FILE_TEST}" >"${TEST_NAME}.stderr" 2>&1
    then
        ok "${TEST_NAME}"
        return
    else
        cat "${TEST_NAME}.stderr" >&2
    fi
    mkdir -p "${TEST_LOG_DIR}"
    cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
    fail "${TEST_NAME}"
}

cmp_json_ok() {
    run_ok "$1" python2 -c "import sys
import re
import json
from cylc.task_state import TASK_STATUSES_ORDERED

# Load data; no need for JSON object try/except as get stderr traceback anyway
output = json.load(open(sys.argv[1], 'r'))
if sys.argv[2]:
    expected_output = json.load(open(sys.argv[2], 'r'))
else:
    expected_output = json.load(sys.stdin)

# Replace arbitary time floats in JSON structures for effective comparison
def float_repl(some_output, marker):
    new_output = re.sub(r'((\'|\")update-time(\'|\")\s*:\s*)\d*\.\d*',
                        r'\1' + marker, str(some_output))
    # Deal with cylc scan TIME_FIELDS (see cylc.state_summary_mgr)
    for status in TASK_STATUSES_ORDERED:
        new_regex = (r'((\'|\")%s(\'|\")\s*:\s*\[\n*\s*\[\n*\s*)\d*\.\d*' %
                     status)
        new_output = re.sub(new_regex, r'\1' + marker, str(new_output))
    return new_output

# Unicode (u'...') for 'loaded' outputs; double-quoted strings for 'dumps'
u_marker = \"u'<FLOAT_REPLACED>'\"
s_marker = '\"<FLOAT_REPLACED>\"'

# Check for match; if different, process for easy stderr lifting -> tests
if float_repl(output, u_marker) == float_repl(expected_output, u_marker):
    sys.exit(0)
else:
    sys.exit('LOADED:\n%s\n!=\n%s\n OR DUMPED:\n%s\n!=\n%s' % (
        float_repl(output, u_marker), float_repl(expected_output, u_marker),
        float_repl(json.dumps(output, indent=4), s_marker),
        float_repl(json.dumps(expected_output, indent=4),
                   s_marker)))" "$2" "${3:-}"
}

contains_ok() {
    local FILE_TEST="$1"
    local FILE_CONTROL="${2:--}"
    local TEST_NAME="$(basename "${FILE_TEST}")-contains-ok"
    comm -13 <(sort "${FILE_TEST}") <(sort "${FILE_CONTROL}") \
        1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"
    if [[ -s "${TEST_NAME}.stdout" ]]; then
        mkdir -p "${TEST_LOG_DIR}"
        echo "Missing lines:" >>"${TEST_NAME}.stderr"
        cat "${TEST_NAME}.stdout" >>"${TEST_NAME}.stderr"
        cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
        fail "${TEST_NAME}"
        return
    fi
    ok "${TEST_NAME}"
}

count_ok() {
    local BRE="$1"
    local FILE="$2"
    local COUNT="$3"
    local TEST_NAME="$(basename "${FILE}")-count-ok"
    local NEW_COUNT=$(grep -c "${BRE}" "${FILE}")
    if (( NEW_COUNT == COUNT )); then
        ok "${TEST_NAME}"
        return
    fi
    mkdir -p "${TEST_LOG_DIR}"
    echo "Found ${NEW_COUNT} (not ${COUNT}) of ${BRE} in ${FILE}" \
        >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
    fail "${TEST_NAME}"
}

grep_ok() {
    local BRE="$1"
    local FILE="$2"
    local TEST_NAME="$(basename "${FILE}")-grep-ok"
    if grep -q -e "${BRE}" "${FILE}"; then
        ok "${TEST_NAME}"
        return
    fi
    mkdir -p "${TEST_LOG_DIR}"
    echo "Can't find ${BRE} in ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
    fail "${TEST_NAME}"
}

grep_fail() {
    local BRE="$1"
    local FILE="$2"
    local TEST_NAME="$(basename "${FILE}")-grep-fail"
    if grep -q -e "${BRE}" "${FILE}"; then
        mkdir -p "${TEST_LOG_DIR}"
        echo "ERROR: Found ${BRE} in ${FILE}" \
          >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
        fail "${TEST_NAME}"
        return
    fi
    ok "${TEST_NAME}"
}

exists_ok() {
    local FILE=$1
    local TEST_NAME="$(basename "${FILE}")-file-exists-ok"
    if [[ -a $FILE ]]; then
        ok "${TEST_NAME}"
        return
    fi
    mkdir -p "${TEST_LOG_DIR}"
    echo "Does not exist: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
    fail "${TEST_NAME}"
}

exists_fail() {
    local FILE="$1"
    local TEST_NAME="$(basename "${FILE}")-file-exists-fail"
    if [[ ! -a "${FILE}" ]]; then
        ok "${TEST_NAME}"
        return
    fi
    mkdir -p "${TEST_LOG_DIR}"
    echo "Exists: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
    fail "${TEST_NAME}"
}

graph_suite() {
    # Generate a graphviz "plain" format graph of a suite.
    local SUITE_NAME="${1}"
    local OUTPUT_FILE="${2}"
    shift 2
    mkdir -p "$(dirname "${OUTPUT_FILE}")"
    cylc graph --reference "${SUITE_NAME}" "$@" >"${OUTPUT_FILE}" 2>'/dev/null'
}

init_suite() {
    local TEST_NAME="$1"
    local SUITE_RC="${2:--}"
    SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
    SUITE_RUN_DIR="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}"
    mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
    cat "${SUITE_RC}" >"${TEST_DIR}/${SUITE_NAME}/suite.rc"
    cylc register "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
    cd "${TEST_DIR}/${SUITE_NAME}"
}

install_suite() {
    local TEST_NAME="$1"
    local TEST_SOURCE_BASE="$2"
    ORIG_SUITE_NAME="${TEST_SOURCE_BASE}"
    SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
    SUITE_RUN_DIR="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}"
    mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
    cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${SUITE_NAME}/"
    cylc register "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
    cd "${TEST_DIR}/${SUITE_NAME}"
}

log_scan () {
    local TEST_NAME="$1"
    local FILE="$2"
    local REPS=$3
    local DELAY=$4
    if ${CYLC_TEST_DEBUG:-false}; then
        local ERR=2
    else
        local ERR=1
    fi
    shift 4
    local count=0
    local success=false
    local position=0
    local newposition=
    for pattern in "$@"; do
        count=$(( count + 1 ))
        success=false
        echo -n "scanning for '${pattern:0:30}'" >& $ERR
        for try in $(seq 1 $REPS); do
            echo -n '.' >& $ERR
            newposition=$(grep -n "$pattern" "$FILE" | \
                tail -n 1 | cut -d ':' -f 1)
            if (( newposition > position )); then
                position=$newposition
                echo ': succeeded' >& $ERR
                ok "${TEST_NAME}-${count}"
                success=true
                break
            fi
            sleep $DELAY
        done
        shift
        if ! $success; then
            echo ': failed' >& $ERR
            fail "${TEST_NAME}-${count}"
            if ${CYLC_TEST_DEBUG:-}; then
                cat "${FILE}" >&2
            fi
            skip "$#"
            return 1
        fi
    done
}

purge_suite_remote() {
    local HOST="$1"  # or "owner@host"
    local NAME="$2"
    local CMD="cd 'cylc-run' && rm -fr '${NAME}'"
    local DIRNAME="$(dirname "${NAME}")"
    if [[ "${DIRNAME}" != '.' ]]; then
        CMD="${CMD} && rmdir -p '${DIRNAME}'"
    fi
    ssh -n -oBatchMode=yes -oConnectTimeout=5 "${HOST}" "${CMD}" \
        2>'/dev/null' || true
}

purge_suite() {
    local SUITE_NAME="$1"
    cd "${TEST_DIR:-}"
    if (($FAILURES == 0)); then
        local RUND="$(cylc get-global-config --print-run-dir)"
        local SUITE_DIR="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}"
        # Note: lsof can hang, so call with "timeout".
        while grep -q "${SUITE_DIR}" < <(timeout 5 lsof); do
            sleep 1
        done
        rm -fr "${SUITE_DIR}"
        (cd "${RUND}" && rmdir -p "$(dirname "${SUITE_DIR}")" 2>'/dev/null' || true)
    fi
    if [[ -n "${TEST_DIR:-}" && -n "${SUITE_NAME:-}" ]]; then
        rm -fr "${TEST_DIR}/${SUITE_NAME}"
    fi
}

poll() {
    local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
    while (($(date +%s) < TIMEOUT)) && eval "$@"; do
        sleep 1
    done
}

skip() {
    local N_TO_SKIP="$1"
    shift 1
    local COUNT=0
    while ((COUNT++ < N_TO_SKIP)); do
        echo "ok $((++TEST_NUMBER)) # skip $@"
    done
}

skip_all() {
    echo "1..0 # SKIP $@"
    exit
}

ssh_install_cylc() {
    local RHOST="$1"
    local RHOST_CYLC_DIR=$(_ssh_mkdtemp_cylc_dir "${RHOST}")
    TEST_RHOST_CYLC_DIR="${RHOST}:${RHOST_CYLC_DIR}"
    rsync -a '--exclude=*.pyc' "${CYLC_DIR}"/* "${RHOST}:${RHOST_CYLC_DIR}/"
    ssh -n -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" \
        "make -C '${RHOST_CYLC_DIR}' 'version'" 1>'/dev/null' 2>&1
}

_ssh_mkdtemp_cylc_dir() {
    local RHOST="$1"
    ssh -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" python2 - <<'__PYTHON__'
import os
from tempfile import mkdtemp
print mkdtemp(dir=os.path.expanduser("~"), prefix="cylc-")
__PYTHON__
}

mock_smtpd_init() {  # Logic borrowed from Rose
    local SMTPD_PORT=
    for SMTPD_PORT in 8025 8125 8225 8325 8425 8525 8625 8725 8825 8925; do 
        local SMTPD_HOST="localhost:${SMTPD_PORT}"
        local SMTPD_LOG="${TEST_DIR}/smtpd.log"
        python2 -m 'smtpd' -c 'DebuggingServer' -d -n "${SMTPD_HOST}" \
            1>"${SMTPD_LOG}" 2>&1 &
        local SMTPD_PID="$!"
        while ! grep -q 'DebuggingServer started' "${SMTPD_LOG}" 2>'/dev/null'
        do
            if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then
                sleep 1
            else
                rm -f "${SMTPD_LOG}"
                unset SMTPD_HOST SMTPD_LOG SMTPD_PID
                break
            fi
        done
        if [[ -n "${SMTPD_PID:-}" ]]; then
            TEST_SMTPD_HOST="${SMTPD_HOST}"
            TEST_SMTPD_PID="${SMTPD_PID}"
            TEST_SMTPD_LOG="${SMTPD_LOG}"
            break
        fi
    done
}

mock_smtpd_kill() {  # Logic borrowed from Rose
    if [[ -n "${TEST_SMTPD_PID:-}" ]] && ps "${TEST_SMTPD_PID}" >'/dev/null' 2>&1
    then
        kill "${TEST_SMTPD_PID}"
        wait "${TEST_SMTPD_PID}" 2>/dev/null || true
        unset TEST_SMTPD_HOST TEST_SMTPD_LOG TEST_SMTPD_PID
    fi
}

create_test_globalrc() {
    # (Documentated in file header).
    local PRE=
    local POST=
    if (( $# == 1 )); then
        PRE=$1
    elif (( $# == 2 )); then
        PRE=$1
        POST=$2
    elif (( $# > 2 )); then
        echo 'ERROR, create_test_globalrc: too many args' >&2
        exit 1
    fi
    # Tidy in case of previous use of this function.
    rm -fr 'etc'
    mkdir 'etc'
    # Suite host self-identification method.
    echo "$PRE" >'etc/global.rc'
    TESTS_CONF_FILE="$(cylc get-global --print-site-dir)/global-tests.rc"
    if [[ -f "${TESTS_CONF_FILE}" ]]; then
        cat "${TESTS_CONF_FILE}" >>'etc/global.rc'
    fi
    echo "$POST" >>'etc/global.rc'
    export CYLC_CONF_PATH="${PWD}/etc"
}

set_test_remote_host() {
  # For remote job tests, define CYLC_TEST_HOST from global config '[test
  # battery]remote host', skipping tests if not defined. Call immediately after
  # sourcing the test header, for skip_all to work.
  # TODO - these tests should be modified to use set_test_remote() instead, to 
  # allow full remote job tests with another user account on the suite host.
  CYLC_TEST_HOST="$( \
      cylc get-global-config -i '[test battery]remote host' 2>'/dev/null')"
  if [[ -z "${CYLC_TEST_HOST}" ]]; then
      skip_all '"[test battery]remote host": not defined'
  fi
  export CYLC_TEST_HOST
}

set_test_remote() {
  # For remote job tests, define CYLC_TEST_HOST and CYLC_TEST_OWNER from global
  # config '[test battery]remote host/owner', defaulting to localhost and $USER.
  # Skip all tests if neither host or owner is defined for a remote account.
  # Call immediately after sourcing the test header, for skip_all to work.
  CYLC_TEST_HOST="$( \
      cylc get-global-config -i '[test battery]remote host' 2>'/dev/null')"
  CYLC_TEST_OWNER="$( \
      cylc get-global-config -i '[test battery]remote owner' 2>'/dev/null')"
  if [[ -z "${CYLC_TEST_HOST}${CYLC_TEST_OWNER}" ]]; then
      skip_all '"[test battery]remote host/owner": not defined'
  fi
  export CYLC_TEST_HOST=${CYLC_TEST_HOST:-"localhost"}
  export CYLC_TEST_OWNER=${CYLC_TEST_OWNER:-${USER}}
}


# tags

port_is_busy() {
    local WS_PORT="${1}"
    if type -P netcat 1>/dev/null; then
        WS_HOSTNAME="${WS_HOSTNAME:-"$(hostname)"}"
        WS_HOSTNAME="${WS_HOSTNAME:-'localhost'}"
        netcat -z "${WS_HOSTNAME}" "${WS_PORT}"
    else
        netstat -atun | grep -q "0.0.0.0:${WS_PORT}"
    fi
}

TEST_CYLC_WS_PID=
TEST_CYLC_WS_PORT=
TEST_CYLC_WS_URL=
cylc_ws_init() {
    local NS="$1"
    local UTIL="$2"

    local WS_PORT="$((${RANDOM} + 10000))"
    while port_is_busy "${WS_PORT}"; do
        WS_PORT="$((${RANDOM} + 10000))"
    done

    TEST_NAME="${TEST_NAME_BASE}-${NS}-${UTIL}"
    "${NS}" "${UTIL}" 'start' "${WS_PORT}"  \
        0<'/dev/null' 1>"${NS}-${UTIL}.stdout" 2>"${NS}-${UTIL}.stderr" &
    TEST_CYLC_WS_PID="$!"
    T_INIT="$(date '+%s')"
    while ! port_is_busy "${WS_PORT}" && (($(date '+%s') < ${T_INIT} + 30)); do
        sleep 1
    done
    TEST_CYLC_WS_PORT="${WS_PORT}"
    if port_is_busy "${TEST_CYLC_WS_PORT}"; then
        ok "${TEST_NAME}"
        TEST_CYLC_WS_URL="http://${HOSTNAME}:${TEST_CYLC_WS_PORT}/"
    else
        fail "${TEST_NAME}"
        cylc_ws_kill
    fi
}

cylc_ws_kill() {
    if [[ -n "${TEST_CYLC_WS_PID}" ]]; then
        kill "${TEST_CYLC_WS_PID}" 2>'/dev/null'
        wait 2>'/dev/null'
    fi
    if [[ -n "${TEST_CYLC_WS_PORT}" ]]; then
        rm -fr "${HOME}/.cylc/"*"-0.0.0.0-${TEST_CYLC_WS_PORT}"* 2>'/dev/null'
    fi
    TEST_CYLC_WS_PID=
    TEST_CYLC_WS_PORT=
    TEST_CYLC_WS_URL=
}

cylc_ws_json_greps() {
    local TEST_NAME="$1"
    shift 1
    run_ok "${TEST_NAME}" python2 - "$@" <<'__PYTHON__'
import ast
import json
import sys

data = json.load(open(sys.argv[1]))
for item in sys.argv[2:]:
    keys, value = ast.literal_eval(item)
    datum = data
    try:
        for key in keys:
            if isinstance(key, dict):
                for datum_item in datum:
                    if all([datum_item.get(k) == v for k, v in key.items()]):
                        datum = datum_item
                        break
                else:
                    raise KeyError
            else:
                datum = datum[key]
        if datum != value:
            raise ValueError((item, datum))
    except IndexError:
        raise IndexError(item)
    except KeyError:
        raise KeyError(item)
__PYTHON__
    if [[ -s "${TEST_NAME}.stderr" ]]; then
        # Only print the final exception line - change to cat for details
        tail -n 1 "${TEST_NAME}.stderr" >&2
    fi
}


CYLC_DIR="${CYLC_DIR:-$(cd $(dirname "${BASH_SOURCE}")/../../.. && pwd)}"
PATH="${CYLC_DIR}/bin:${PATH}"

TEST_NAME_BASE="$(basename "$0" '.t')"
TEST_SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)"
TEST_DIR="$(mktemp -d)"
cd "${TEST_DIR}"
TEST_SOURCE_DIR_BASE="${TEST_SOURCE_DIR##*tests/}"
TEST_LOG_DIR_BASE="${TMPDIR:-/tmp}/${USER}/cylctb-${CYLC_TEST_TIME_INIT}"
TEST_LOG_DIR="${TEST_LOG_DIR_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}"
SUITE_RUN_FAILS=false
SSH_OPTS='-oBatchMode=yes -oConnectTimeout=10'

CYLC_TEST_SKIP="${CYLC_TEST_SKIP:-}"
# Is this test in the skip list?
THIS="${0#./}"
THIS_DIR="$(dirname "${THIS}")"
for SKIP in ${CYLC_TEST_SKIP}; do
    RSKIP="${SKIP#./}"
    if [[ "${THIS}" == "${RSKIP}" || "${THIS_DIR%/}" == "${RSKIP%/}" ]]; then
        skip_all 'this test is in $CYLC_TEST_SKIP.'
        break
    fi
done
if ! "${CYLC_TEST_RUN_GENERIC}" && "${CYLC_TEST_IS_GENERIC}"; then
    skip_all 'not running generic tests'
elif ! "${CYLC_TEST_RUN_PLATFORM}" && ! "${CYLC_TEST_IS_GENERIC}"; then
    skip_all 'not running platform-specific tests'
fi
# Ignore the normal site/user global config, use etc/global-tests.rc.
create_test_globalrc
set +x
set +e
