JaySynth/tools/testhost/tests/run_tests.sh

#!/usr/bin/env bash
#
# Regression suite for the findings tracked in JaySynth's TODO.md, run via
# the testhost tool itself. See tests/README.md for what each test actually
# covers, and - just as importantly - what it doesn't.
#
# Usage:
#   MAKE_HOME=/path/to/submodule/make ./run_tests.sh [path/to/JaySynth.so]
#
# Exit code is 0 if every test passed, 1 otherwise (so this is CI-friendly).

set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TESTHOST_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
REPO_ROOT="$(cd "$TESTHOST_DIR/../.." && pwd)"

TESTHOST_BIN="$TESTHOST_DIR/build/linux/release/testhost"
PLUGIN="${1:-$REPO_ROOT/build/linux/release/JaySynth.so}"

WORKDIR="$(mktemp -d /tmp/testhost-suite.XXXXXX)"
PASS_COUNT=0
FAIL_COUNT=0
FAILED_NAMES=()

log()  { printf '%s\n' "$*"; }
pass() { log "  PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); }
fail() { log "  FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); FAILED_NAMES+=("$1"); }

# --- Preconditions -----------------------------------------------------

if [ ! -x "$TESTHOST_BIN" ]; then
	log "Building testhost..."
	if ! MAKE_HOME="${MAKE_HOME:-$(realpath "$REPO_ROOT/submodule/make")}" make -C "$TESTHOST_DIR" > "$WORKDIR/build.log" 2>&1; then
		log "FATAL: testhost build failed - see $WORKDIR/build.log"
		exit 1
	fi
fi

if [ ! -f "$PLUGIN" ]; then
	log "FATAL: plugin not found at '$PLUGIN' (build JaySynth first, or pass its path as \$1)"
	exit 1
fi

run_testhost() {
	# run_testhost <timeout_seconds> <args...>
	local timeout_s="$1"; shift
	timeout "${timeout_s}s" "$TESTHOST_BIN" --plugin "$PLUGIN" "$@"
}

wav_is_valid_and_nonsilent() {
	# Very small, dependency-light sanity check: file exists, non-trivial
	# size (a 0/near-0 byte file means recording never really started).
	[ -s "$1" ] && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -gt 1000 ]
}

# --- Test 1: oversized block size (Critical #5) -------------------------
# Before the fix, a host block size larger than SYNTH_MAX_BUFSIZE (8192)
# either overflowed processBlock's fixed 8192-sample stack buffer or
# deadlocked the render worker threads (JaySynthThread::go() silently
# no-op'd, and renderNextBlock's wait(-1) blocked forever). The `timeout`
# wrapper is what actually catches the deadlock case - a hang would
# otherwise just sit here forever.
test_oversized_blocksize() {
	local name="oversized_blocksize (Critical #5)"
	local out="$WORKDIR/oversized.wav"
	local log_file="$WORKDIR/oversized.log"

	if run_testhost 20 --blockSize 16384 --note 60 100 --duration 1.5 --wav "$out" > "$log_file" 2>&1; then
		if grep -qi "WARNING - output contained NaN" "$log_file"; then
			fail "$name (rendered but produced NaN/Inf - see $log_file)"
		elif ! wav_is_valid_and_nonsilent "$out"; then
			fail "$name (no valid WAV produced - see $log_file)"
		else
			pass "$name"
		fi
	else
		local rc=$?
		if [ "$rc" -eq 124 ]; then
			fail "$name (HUNG - killed by timeout, matches the pre-fix deadlock)"
		else
			fail "$name (crashed, exit code $rc - see $log_file)"
		fi
	fi
}

# --- Test 2/3: NRPN controller ID bounds (Critical #1) -------------------
# handleController used to write last_midiCC_info[midiCC_info.ID] with no
# bounds check; NRPN IDs assemble to up to 16383 (14-bit) but the array is
# sized NUM_MIDI_CONTROLLERS=1024. This can't be observed from the outside
# without a memory sanitizer (the write lands inside the same heap object,
# not necessarily past its allocation), so "doesn't crash" is a necessary
# but not fully sufficient check - the real guarantee is the bounds-check
# line itself (see tests/README.md). Still worth running both an
# out-of-range and an in-range NRPN sequence, so a regression that made the
# check *too* aggressive (rejecting valid IDs) would also show up as a
# behavioural difference between the two runs.
test_nrpn_bounds() {
	local name_oor="nrpn_out_of_range, ID=16383 (Critical #1)"
	local name_ir="nrpn_in_range, ID=500 (Critical #1 regression guard)"

	local out_oor="$WORKDIR/nrpn_oor.wav"
	local log_oor="$WORKDIR/nrpn_oor.log"
	if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_out_of_range.json" > "$log_oor" 2>&1; then
		# recordWav in the scenario JSON resolves relative to the scenario
		# file's own directory, not $WORKDIR - move it out after the run.
		mv "$SCRIPT_DIR/scenarios/out_nrpn_out_of_range.wav" "$out_oor" 2>/dev/null
		if grep -qi "WARNING - output contained NaN" "$log_oor"; then
			fail "$name_oor (NaN/Inf in output - see $log_oor)"
		elif ! wav_is_valid_and_nonsilent "$out_oor"; then
			fail "$name_oor (no valid WAV - see $log_oor)"
		else
			pass "$name_oor"
		fi
	else
		local rc=$?
		fail "$name_oor (exit code $rc, expected clean completion - see $log_oor)"
	fi

	local out_ir="$WORKDIR/nrpn_ir.wav"
	local log_ir="$WORKDIR/nrpn_ir.log"
	if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_in_range.json" > "$log_ir" 2>&1; then
		mv "$SCRIPT_DIR/scenarios/out_nrpn_in_range.wav" "$out_ir" 2>/dev/null
		if grep -qi "WARNING - output contained NaN" "$log_ir"; then
			fail "$name_ir (NaN/Inf in output - see $log_ir)"
		elif ! wav_is_valid_and_nonsilent "$out_ir"; then
			fail "$name_ir (no valid WAV - see $log_ir)"
		else
			pass "$name_ir"
		fi
	else
		local rc=$?
		fail "$name_ir (exit code $rc - see $log_ir)"
	fi
}

# --- Test 4: patch state save/load round-trip ---------------------------
# Covers two related fixes together: (a) setCurrentProgramStateInformation
# was passing patchImportXml an XML node one level too deep, making the
# VST2 "copy plugin state" path a complete no-op (found while fixing the
# Patch/Bank import-export findings); (b) patchDecodeXml/patchDecodeXml_legacy
# deduplication and the patchImportXml legacy-branch restructuring. All of
# these sit on the exact save->load call chain exercised here. Uses a
# continuous parameter (index 0 = SYNTH_PARAM_VOLUME) rather than a
# stepped/boolean one, and a tolerance rather than exact equality, because
# the value/slider scaling curve (toParam/toSlider) is not bit-exact across
# a round trip - see tests/README.md.
test_patch_roundtrip() {
	local name="patch_save_load_roundtrip"
	local patch_file="$WORKDIR/roundtrip_patch.bin"
	local log1="$WORKDIR/roundtrip_save.log"
	local log2="$WORKDIR/roundtrip_load.log"
	local target_value="0.37"
	local tolerance="0.02"

	if ! run_testhost 10 --param 0 "$target_value" --savePatch "$patch_file" --duration 0.05 > "$log1" 2>&1; then
		fail "$name (save step failed, exit $? - see $log1)"
		return
	fi

	if [ ! -s "$patch_file" ]; then
		fail "$name (no patch file written)"
		return
	fi

	if ! run_testhost 10 --patch "$patch_file" --printParams --duration 0.01 > "$log2" 2>&1; then
		fail "$name (load step failed, exit $? - see $log2)"
		return
	fi

	local readback
	readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')"

	if [ -z "$readback" ]; then
		fail "$name (PARAM 0 not found in output - see $log2)"
		return
	fi

	local diff
	diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')"
	local within_tolerance
	within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')"

	if [ "$within_tolerance" = "1" ]; then
		pass "$name (wrote $target_value, read back $readback, within tolerance)"
	else
		fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)"
	fi
}

# --- Test 5: bank state save/load round-trip -----------------------------
# Same idea as test 4 but through getStateInformation/setStateInformation
# (the whole-bank path) and bankEncodeXml/bankDecodeXml/bankImportXml,
# rather than the single-patch path.
test_bank_roundtrip() {
	local name="bank_save_load_roundtrip"
	local bank_file="$WORKDIR/roundtrip_bank.bin"
	local log1="$WORKDIR/roundtrip_bank_save.log"
	local log2="$WORKDIR/roundtrip_bank_load.log"
	local target_value="0.62"
	local tolerance="0.02"

	if ! run_testhost 10 --param 0 "$target_value" --saveBank "$bank_file" --duration 0.05 > "$log1" 2>&1; then
		fail "$name (save step failed, exit $? - see $log1)"
		return
	fi

	if [ ! -s "$bank_file" ]; then
		fail "$name (no bank file written)"
		return
	fi

	if ! run_testhost 10 --bank "$bank_file" --printParams --duration 0.01 > "$log2" 2>&1; then
		fail "$name (load step failed, exit $? - see $log2)"
		return
	fi

	local readback
	readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')"

	if [ -z "$readback" ]; then
		fail "$name (PARAM 0 not found in output - see $log2)"
		return
	fi

	local diff
	diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')"
	local within_tolerance
	within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')"

	if [ "$within_tolerance" = "1" ]; then
		pass "$name (wrote $target_value, read back $readback, within tolerance)"
	else
		fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)"
	fi
}

# --- Test 6: stability sweep over bundled sample patches ------------------
# Broad regression net: load every bundled .fxp under extras/sounds/ and
# render a held note through it. NOTE: as discovered while building this
# suite, these particular sample files are in VST2's opaque-chunk ("FPCh")
# format, but JUCE 3.1.1's plugin-side VST wrapper (compiled into
# JaySynth.so) never sets effFlagsProgramChunks, so JUCE's *host*-side
# VSTPluginInstance::usesChunks() sees false and silently ignores the
# chunk data entirely - confirmed by diffing --printParams output with and
# without --patch: zero parameters differ. This is a real, separate
# finding (see tests/README.md and testhost/TODO.md) - NOT one of the
# tracked TODO.md findings, and NOT something this suite attempts to fix.
# So this test only checks "loading this file doesn't crash the process",
# not "loading this file changes the patch" - it still exercises the
# malloc/delete[] paths touched by the Memory-management fixes as a broad
# smoke test, just not the patch-content-application path.
test_stability_sweep() {
	local sounds_dir="$REPO_ROOT/extras/sounds"
	local any=0

	while IFS= read -r -d '' fxp; do
		any=1
		local base
		base="$(basename "$fxp")"
		local log_file="$WORKDIR/sweep_$(echo "$base" | tr -c 'A-Za-z0-9._-' '_').log"

		if run_testhost 15 --patch "$fxp" --note 60 100 --duration 0.5 > "$log_file" 2>&1; then
			if grep -qi "WARNING - output contained NaN" "$log_file"; then
				fail "stability_sweep: '$base' (NaN/Inf - see $log_file)"
			else
				pass "stability_sweep: '$base'"
			fi
		else
			local rc=$?
			fail "stability_sweep: '$base' (exit code $rc, expected 0 - see $log_file)"
		fi
	done < <(find "$sounds_dir" -maxdepth 1 -iname '*.fxp' -print0 2>/dev/null)

	if [ "$any" -eq 0 ]; then
		log "  (no .fxp files found under $sounds_dir - skipping stability sweep)"
	fi
}

# --- Run everything -------------------------------------------------------

log "testhost regression suite"
log "  testhost:  $TESTHOST_BIN"
log "  plugin:    $PLUGIN"
log "  workdir:   $WORKDIR"
log ""

log "Critical finding #5 - oversized block size:"
test_oversized_blocksize
log ""

log "Critical finding #1 - NRPN controller ID bounds:"
test_nrpn_bounds
log ""

log "Patch/Bank import-export fixes - state round-trip:"
test_patch_roundtrip
test_bank_roundtrip
log ""

log "General stability sweep over bundled sample patches:"
test_stability_sweep
log ""

log "----------------------------------------"
log "Results: $PASS_COUNT passed, $FAIL_COUNT failed"
if [ "$FAIL_COUNT" -gt 0 ]; then
	log "Failed:"
	for n in "${FAILED_NAMES[@]}"; do
		log "  - $n"
	done
	log "Artifacts (logs/WAVs) kept in: $WORKDIR"
	exit 1
fi

log "All tests passed. Cleaning up $WORKDIR."
rm -rf "$WORKDIR"
exit 0