#!/usr/bin/env bash # # Regression suite for the findings tracked in JaySynth's TODO.md, run via # the testhost tool itself. See tests/README.md for what each test actually # covers, and - just as importantly - what it doesn't. # # Usage: # MAKE_HOME=/path/to/submodule/make ./run_tests.sh [path/to/JaySynth.so] # # Exit code is 0 if every test passed, 1 otherwise (so this is CI-friendly). set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TESTHOST_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" REPO_ROOT="$(cd "$TESTHOST_DIR/../.." && pwd)" TESTHOST_BIN="$TESTHOST_DIR/build/linux/release/testhost" PLUGIN="${1:-$REPO_ROOT/build/linux/release/JaySynth.so}" WORKDIR="$(mktemp -d /tmp/testhost-suite.XXXXXX)" PASS_COUNT=0 FAIL_COUNT=0 FAILED_NAMES=() log() { printf '%s\n' "$*"; } pass() { log " PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); } fail() { log " FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); FAILED_NAMES+=("$1"); } # --- Preconditions ----------------------------------------------------- if [ ! -x "$TESTHOST_BIN" ]; then log "Building testhost..." if ! MAKE_HOME="${MAKE_HOME:-$(realpath "$REPO_ROOT/submodule/make")}" make -C "$TESTHOST_DIR" > "$WORKDIR/build.log" 2>&1; then log "FATAL: testhost build failed - see $WORKDIR/build.log" exit 1 fi fi if [ ! -f "$PLUGIN" ]; then log "FATAL: plugin not found at '$PLUGIN' (build JaySynth first, or pass its path as \$1)" exit 1 fi run_testhost() { # run_testhost local timeout_s="$1"; shift timeout "${timeout_s}s" "$TESTHOST_BIN" --plugin "$PLUGIN" "$@" } wav_is_valid_and_nonsilent() { # Very small, dependency-light sanity check: file exists, non-trivial # size (a 0/near-0 byte file means recording never really started). [ -s "$1" ] && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -gt 1000 ] } # --- Test 1: oversized block size (Critical #5) ------------------------- # Before the fix, a host block size larger than SYNTH_MAX_BUFSIZE (8192) # either overflowed processBlock's fixed 8192-sample stack buffer or # deadlocked the render worker threads (JaySynthThread::go() silently # no-op'd, and renderNextBlock's wait(-1) blocked forever). The `timeout` # wrapper is what actually catches the deadlock case - a hang would # otherwise just sit here forever. test_oversized_blocksize() { local name="oversized_blocksize (Critical #5)" local out="$WORKDIR/oversized.wav" local log_file="$WORKDIR/oversized.log" if run_testhost 20 --blockSize 16384 --note 60 100 --duration 1.5 --wav "$out" > "$log_file" 2>&1; then if grep -qi "WARNING - output contained NaN" "$log_file"; then fail "$name (rendered but produced NaN/Inf - see $log_file)" elif ! wav_is_valid_and_nonsilent "$out"; then fail "$name (no valid WAV produced - see $log_file)" else pass "$name" fi else local rc=$? if [ "$rc" -eq 124 ]; then fail "$name (HUNG - killed by timeout, matches the pre-fix deadlock)" else fail "$name (crashed, exit code $rc - see $log_file)" fi fi } # --- Test 2/3: NRPN controller ID bounds (Critical #1) ------------------- # handleController used to write last_midiCC_info[midiCC_info.ID] with no # bounds check; NRPN IDs assemble to up to 16383 (14-bit) but the array is # sized NUM_MIDI_CONTROLLERS=1024. This can't be observed from the outside # without a memory sanitizer (the write lands inside the same heap object, # not necessarily past its allocation), so "doesn't crash" is a necessary # but not fully sufficient check - the real guarantee is the bounds-check # line itself (see tests/README.md). Still worth running both an # out-of-range and an in-range NRPN sequence, so a regression that made the # check *too* aggressive (rejecting valid IDs) would also show up as a # behavioural difference between the two runs. test_nrpn_bounds() { local name_oor="nrpn_out_of_range, ID=16383 (Critical #1)" local name_ir="nrpn_in_range, ID=500 (Critical #1 regression guard)" local out_oor="$WORKDIR/nrpn_oor.wav" local log_oor="$WORKDIR/nrpn_oor.log" if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_out_of_range.json" > "$log_oor" 2>&1; then # recordWav in the scenario JSON resolves relative to the scenario # file's own directory, not $WORKDIR - move it out after the run. mv "$SCRIPT_DIR/scenarios/out_nrpn_out_of_range.wav" "$out_oor" 2>/dev/null if grep -qi "WARNING - output contained NaN" "$log_oor"; then fail "$name_oor (NaN/Inf in output - see $log_oor)" elif ! wav_is_valid_and_nonsilent "$out_oor"; then fail "$name_oor (no valid WAV - see $log_oor)" else pass "$name_oor" fi else local rc=$? fail "$name_oor (exit code $rc, expected clean completion - see $log_oor)" fi local out_ir="$WORKDIR/nrpn_ir.wav" local log_ir="$WORKDIR/nrpn_ir.log" if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_in_range.json" > "$log_ir" 2>&1; then mv "$SCRIPT_DIR/scenarios/out_nrpn_in_range.wav" "$out_ir" 2>/dev/null if grep -qi "WARNING - output contained NaN" "$log_ir"; then fail "$name_ir (NaN/Inf in output - see $log_ir)" elif ! wav_is_valid_and_nonsilent "$out_ir"; then fail "$name_ir (no valid WAV - see $log_ir)" else pass "$name_ir" fi else local rc=$? fail "$name_ir (exit code $rc - see $log_ir)" fi } # --- Test 4: patch state save/load round-trip --------------------------- # Covers two related fixes together: (a) setCurrentProgramStateInformation # was passing patchImportXml an XML node one level too deep, making the # VST2 "copy plugin state" path a complete no-op (found while fixing the # Patch/Bank import-export findings); (b) patchDecodeXml/patchDecodeXml_legacy # deduplication and the patchImportXml legacy-branch restructuring. All of # these sit on the exact save->load call chain exercised here. Uses a # continuous parameter (index 0 = SYNTH_PARAM_VOLUME) rather than a # stepped/boolean one, and a tolerance rather than exact equality, because # the value/slider scaling curve (toParam/toSlider) is not bit-exact across # a round trip - see tests/README.md. test_patch_roundtrip() { local name="patch_save_load_roundtrip" local patch_file="$WORKDIR/roundtrip_patch.bin" local log1="$WORKDIR/roundtrip_save.log" local log2="$WORKDIR/roundtrip_load.log" local target_value="0.37" local tolerance="0.02" if ! run_testhost 10 --param 0 "$target_value" --savePatch "$patch_file" --duration 0.05 > "$log1" 2>&1; then fail "$name (save step failed, exit $? - see $log1)" return fi if [ ! -s "$patch_file" ]; then fail "$name (no patch file written)" return fi if ! run_testhost 10 --patch "$patch_file" --printParams --duration 0.01 > "$log2" 2>&1; then fail "$name (load step failed, exit $? - see $log2)" return fi local readback readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')" if [ -z "$readback" ]; then fail "$name (PARAM 0 not found in output - see $log2)" return fi local diff diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')" local within_tolerance within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')" if [ "$within_tolerance" = "1" ]; then pass "$name (wrote $target_value, read back $readback, within tolerance)" else fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)" fi } # --- Test 5: bank state save/load round-trip ----------------------------- # Same idea as test 4 but through getStateInformation/setStateInformation # (the whole-bank path) and bankEncodeXml/bankDecodeXml/bankImportXml, # rather than the single-patch path. test_bank_roundtrip() { local name="bank_save_load_roundtrip" local bank_file="$WORKDIR/roundtrip_bank.bin" local log1="$WORKDIR/roundtrip_bank_save.log" local log2="$WORKDIR/roundtrip_bank_load.log" local target_value="0.62" local tolerance="0.02" if ! run_testhost 10 --param 0 "$target_value" --saveBank "$bank_file" --duration 0.05 > "$log1" 2>&1; then fail "$name (save step failed, exit $? - see $log1)" return fi if [ ! -s "$bank_file" ]; then fail "$name (no bank file written)" return fi if ! run_testhost 10 --bank "$bank_file" --printParams --duration 0.01 > "$log2" 2>&1; then fail "$name (load step failed, exit $? - see $log2)" return fi local readback readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')" if [ -z "$readback" ]; then fail "$name (PARAM 0 not found in output - see $log2)" return fi local diff diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')" local within_tolerance within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')" if [ "$within_tolerance" = "1" ]; then pass "$name (wrote $target_value, read back $readback, within tolerance)" else fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)" fi } # --- Test 6: stability sweep over bundled sample patches ------------------ # Broad regression net: load every bundled .fxp under extras/sounds/ and # render a held note through it. NOTE: as discovered while building this # suite, these particular sample files are in VST2's opaque-chunk ("FPCh") # format, but JUCE 3.1.1's plugin-side VST wrapper (compiled into # JaySynth.so) never sets effFlagsProgramChunks, so JUCE's *host*-side # VSTPluginInstance::usesChunks() sees false and silently ignores the # chunk data entirely - confirmed by diffing --printParams output with and # without --patch: zero parameters differ. This is a real, separate # finding (see tests/README.md and testhost/TODO.md) - NOT one of the # tracked TODO.md findings, and NOT something this suite attempts to fix. # So this test only checks "loading this file doesn't crash the process", # not "loading this file changes the patch" - it still exercises the # malloc/delete[] paths touched by the Memory-management fixes as a broad # smoke test, just not the patch-content-application path. test_stability_sweep() { local sounds_dir="$REPO_ROOT/extras/sounds" local any=0 while IFS= read -r -d '' fxp; do any=1 local base base="$(basename "$fxp")" local log_file="$WORKDIR/sweep_$(echo "$base" | tr -c 'A-Za-z0-9._-' '_').log" if run_testhost 15 --patch "$fxp" --note 60 100 --duration 0.5 > "$log_file" 2>&1; then if grep -qi "WARNING - output contained NaN" "$log_file"; then fail "stability_sweep: '$base' (NaN/Inf - see $log_file)" else pass "stability_sweep: '$base'" fi else local rc=$? fail "stability_sweep: '$base' (exit code $rc, expected 0 - see $log_file)" fi done < <(find "$sounds_dir" -maxdepth 1 -iname '*.fxp' -print0 2>/dev/null) if [ "$any" -eq 0 ]; then log " (no .fxp files found under $sounds_dir - skipping stability sweep)" fi } # --- Run everything ------------------------------------------------------- log "testhost regression suite" log " testhost: $TESTHOST_BIN" log " plugin: $PLUGIN" log " workdir: $WORKDIR" log "" log "Critical finding #5 - oversized block size:" test_oversized_blocksize log "" log "Critical finding #1 - NRPN controller ID bounds:" test_nrpn_bounds log "" log "Patch/Bank import-export fixes - state round-trip:" test_patch_roundtrip test_bank_roundtrip log "" log "General stability sweep over bundled sample patches:" test_stability_sweep log "" log "----------------------------------------" log "Results: $PASS_COUNT passed, $FAIL_COUNT failed" if [ "$FAIL_COUNT" -gt 0 ]; then log "Failed:" for n in "${FAILED_NAMES[@]}"; do log " - $n" done log "Artifacts (logs/WAVs) kept in: $WORKDIR" exit 1 fi log "All tests passed. Cleaning up $WORKDIR." rm -rf "$WORKDIR" exit 0