#!/usr/bin/env bash # # Regression suite for the findings tracked in JaySynth's TODO.md, run via # the testhost tool itself. See tests/README.md for what each test actually # covers, and - just as importantly - what it doesn't. # # Usage: # MAKE_HOME=/path/to/submodule/make ./run_tests.sh [path/to/JaySynth.so] # # Exit code is 0 if every test passed, 1 otherwise (so this is CI-friendly). set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TESTHOST_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" REPO_ROOT="$(cd "$TESTHOST_DIR/../.." && pwd)" TESTHOST_BIN="$TESTHOST_DIR/build/linux/release/testhost" PLUGIN="${1:-$REPO_ROOT/build/linux/release/JaySynth.so}" WORKDIR="$(mktemp -d /tmp/testhost-suite.XXXXXX)" PASS_COUNT=0 FAIL_COUNT=0 FAILED_NAMES=() log() { printf '%s\n' "$*"; } pass() { log " PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); } fail() { log " FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); FAILED_NAMES+=("$1"); } # --- Preconditions ----------------------------------------------------- if [ ! -x "$TESTHOST_BIN" ]; then log "Building testhost..." if ! MAKE_HOME="${MAKE_HOME:-$(realpath "$REPO_ROOT/submodule/make")}" make -C "$TESTHOST_DIR" > "$WORKDIR/build.log" 2>&1; then log "FATAL: testhost build failed - see $WORKDIR/build.log" exit 1 fi fi if [ ! -f "$PLUGIN" ]; then log "FATAL: plugin not found at '$PLUGIN' (build JaySynth first, or pass its path as \$1)" exit 1 fi run_testhost() { # run_testhost local timeout_s="$1"; shift timeout "${timeout_s}s" "$TESTHOST_BIN" --plugin "$PLUGIN" "$@" } wav_is_valid_and_nonsilent() { # Very small, dependency-light sanity check: file exists, non-trivial # size (a 0/near-0 byte file means recording never really started). [ -s "$1" ] && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -gt 1000 ] } # --- Test 1: oversized block size (Critical #5) ------------------------- # Before the fix, a host block size larger than SYNTH_MAX_BUFSIZE (8192) # either overflowed processBlock's fixed 8192-sample stack buffer or # deadlocked the render worker threads (JaySynthThread::go() silently # no-op'd, and renderNextBlock's wait(-1) blocked forever). The `timeout` # wrapper is what actually catches the deadlock case - a hang would # otherwise just sit here forever. test_oversized_blocksize() { local name="oversized_blocksize (Critical #5)" local out="$WORKDIR/oversized.wav" local log_file="$WORKDIR/oversized.log" if run_testhost 20 --blockSize 16384 --note 60 100 --duration 1.5 --wav "$out" > "$log_file" 2>&1; then if grep -qi "WARNING - output contained NaN" "$log_file"; then fail "$name (rendered but produced NaN/Inf - see $log_file)" elif ! wav_is_valid_and_nonsilent "$out"; then fail "$name (no valid WAV produced - see $log_file)" else pass "$name" fi else local rc=$? if [ "$rc" -eq 124 ]; then fail "$name (HUNG - killed by timeout, matches the pre-fix deadlock)" else fail "$name (crashed, exit code $rc - see $log_file)" fi fi } # --- Test 2/3: NRPN controller ID bounds (Critical #1) ------------------- # handleController used to write last_midiCC_info[midiCC_info.ID] with no # bounds check; NRPN IDs assemble to up to 16383 (14-bit) but the array is # sized NUM_MIDI_CONTROLLERS=1024. This can't be observed from the outside # without a memory sanitizer (the write lands inside the same heap object, # not necessarily past its allocation), so "doesn't crash" is a necessary # but not fully sufficient check - the real guarantee is the bounds-check # line itself (see tests/README.md). Still worth running both an # out-of-range and an in-range NRPN sequence, so a regression that made the # check *too* aggressive (rejecting valid IDs) would also show up as a # behavioural difference between the two runs. test_nrpn_bounds() { local name_oor="nrpn_out_of_range, ID=16383 (Critical #1)" local name_ir="nrpn_in_range, ID=500 (Critical #1 regression guard)" local out_oor="$WORKDIR/nrpn_oor.wav" local log_oor="$WORKDIR/nrpn_oor.log" if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_out_of_range.json" > "$log_oor" 2>&1; then # recordWav in the scenario JSON resolves relative to the scenario # file's own directory, not $WORKDIR - move it out after the run. mv "$SCRIPT_DIR/scenarios/out_nrpn_out_of_range.wav" "$out_oor" 2>/dev/null if grep -qi "WARNING - output contained NaN" "$log_oor"; then fail "$name_oor (NaN/Inf in output - see $log_oor)" elif ! wav_is_valid_and_nonsilent "$out_oor"; then fail "$name_oor (no valid WAV - see $log_oor)" else pass "$name_oor" fi else local rc=$? fail "$name_oor (exit code $rc, expected clean completion - see $log_oor)" fi local out_ir="$WORKDIR/nrpn_ir.wav" local log_ir="$WORKDIR/nrpn_ir.log" if run_testhost 15 --scenario "$SCRIPT_DIR/scenarios/nrpn_in_range.json" > "$log_ir" 2>&1; then mv "$SCRIPT_DIR/scenarios/out_nrpn_in_range.wav" "$out_ir" 2>/dev/null if grep -qi "WARNING - output contained NaN" "$log_ir"; then fail "$name_ir (NaN/Inf in output - see $log_ir)" elif ! wav_is_valid_and_nonsilent "$out_ir"; then fail "$name_ir (no valid WAV - see $log_ir)" else pass "$name_ir" fi else local rc=$? fail "$name_ir (exit code $rc - see $log_ir)" fi } # --- Test 4: patch state save/load round-trip --------------------------- # Covers two related fixes together: (a) setCurrentProgramStateInformation # was passing patchImportXml an XML node one level too deep, making the # VST2 "copy plugin state" path a complete no-op (found while fixing the # Patch/Bank import-export findings); (b) patchDecodeXml/patchDecodeXml_legacy # deduplication and the patchImportXml legacy-branch restructuring. All of # these sit on the exact save->load call chain exercised here. Uses a # continuous parameter (index 0 = SYNTH_PARAM_VOLUME) rather than a # stepped/boolean one, and a tolerance rather than exact equality, because # the value/slider scaling curve (toParam/toSlider) is not bit-exact across # a round trip - see tests/README.md. test_patch_roundtrip() { local name="patch_save_load_roundtrip" local patch_file="$WORKDIR/roundtrip_patch.bin" local log1="$WORKDIR/roundtrip_save.log" local log2="$WORKDIR/roundtrip_load.log" local target_value="0.37" local tolerance="0.02" if ! run_testhost 10 --param 0 "$target_value" --savePatch "$patch_file" --duration 0.05 > "$log1" 2>&1; then fail "$name (save step failed, exit $? - see $log1)" return fi if [ ! -s "$patch_file" ]; then fail "$name (no patch file written)" return fi if ! run_testhost 10 --patch "$patch_file" --printParams --duration 0.01 > "$log2" 2>&1; then fail "$name (load step failed, exit $? - see $log2)" return fi local readback readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')" if [ -z "$readback" ]; then fail "$name (PARAM 0 not found in output - see $log2)" return fi local diff diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')" local within_tolerance within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')" if [ "$within_tolerance" = "1" ]; then pass "$name (wrote $target_value, read back $readback, within tolerance)" else fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)" fi } # --- Test 5: bank state save/load round-trip ----------------------------- # Same idea as test 4 but through getStateInformation/setStateInformation # (the whole-bank path) and bankEncodeXml/bankDecodeXml/bankImportXml, # rather than the single-patch path. test_bank_roundtrip() { local name="bank_save_load_roundtrip" local bank_file="$WORKDIR/roundtrip_bank.bin" local log1="$WORKDIR/roundtrip_bank_save.log" local log2="$WORKDIR/roundtrip_bank_load.log" local target_value="0.62" local tolerance="0.02" if ! run_testhost 10 --param 0 "$target_value" --saveBank "$bank_file" --duration 0.05 > "$log1" 2>&1; then fail "$name (save step failed, exit $? - see $log1)" return fi if [ ! -s "$bank_file" ]; then fail "$name (no bank file written)" return fi if ! run_testhost 10 --bank "$bank_file" --printParams --duration 0.01 > "$log2" 2>&1; then fail "$name (load step failed, exit $? - see $log2)" return fi local readback readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')" if [ -z "$readback" ]; then fail "$name (PARAM 0 not found in output - see $log2)" return fi local diff diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')" local within_tolerance within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')" if [ "$within_tolerance" = "1" ]; then pass "$name (wrote $target_value, read back $readback, within tolerance)" else fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)" fi } # --- Test 6: bank/patch round-trip via the raw VST2 chunk dispatcher ------ # test_bank_roundtrip/test_patch_roundtrip above go through JUCE's generic # AudioProcessor::get/setStateInformation - which, it turns out, JaySynth's # usesChunks() flag genuinely is true for, so those already exercise the # real bankEncodeXml/bankDecodeXml (chunk) path, not a per-parameter # fallback (see tests/README.md's "corrected finding" note - an earlier # version of this comment claimed otherwise and was wrong). This test goes # one step further and calls effGetChunk/effSetChunk *directly* via # Vst2RawChunk (bypassing JUCE's host-side wrapper entirely), so it keeps # testing the real chunk path even if some future JUCE version's # usesChunks() gating ever changes. It also positively *proves* the chunk # path was used - not just that parameters round-trip - by checking the # saved file starts with a zlib stream header (0x78) and is far larger # than a raw 140-parameter float array (140*4=560 bytes) could be; the # regular/fallback format would look nothing like this. test_chunk_roundtrip() { # $1: "bank" or "patch"; $2: target parameter value (kept distinct # per call so a mixup between the two wouldn't silently pass) local kind="$1" local target_value="$2" local save_flag load_flag if [ "$kind" = "bank" ]; then save_flag="--saveBankChunk"; load_flag="--loadBankChunk" else save_flag="--savePatchChunk"; load_flag="--loadPatchChunk" fi local name="${kind}_chunk_roundtrip (raw effGetChunk/effSetChunk, index=$([ "$kind" = bank ] && echo 0 || echo 1))" local chunk_file="$WORKDIR/roundtrip_${kind}.chunk" local log1="$WORKDIR/roundtrip_${kind}_chunk_save.log" local log2="$WORKDIR/roundtrip_${kind}_chunk_load.log" local tolerance="0.02" if ! run_testhost 10 --param 0 "$target_value" $save_flag "$chunk_file" --duration 0.05 > "$log1" 2>&1; then fail "$name (save step failed, exit $? - see $log1)" return fi if [ ! -s "$chunk_file" ]; then fail "$name (no chunk file written)" return fi local first_byte first_byte="$(od -An -tx1 -N1 "$chunk_file" | tr -d ' ')" local chunk_size chunk_size="$(stat -c%s "$chunk_file" 2>/dev/null || echo 0)" if [ "$first_byte" != "78" ]; then fail "$name (saved file doesn't start with a zlib header (0x78) - got 0x$first_byte, this didn't go through bankEncodeXml/GZIP at all)" return fi if [ "$chunk_size" -le 1000 ]; then fail "$name (saved file is only $chunk_size bytes - too small to be the real XML/chunk format, looks like a per-parameter fallback)" return fi if ! run_testhost 10 $load_flag "$chunk_file" --printParams --duration 0.01 > "$log2" 2>&1; then fail "$name (load step failed, exit $? - see $log2)" return fi local readback readback="$(grep '^PARAM 0 ' "$log2" | awk '{print $3}')" if [ -z "$readback" ]; then fail "$name (PARAM 0 not found in output - see $log2)" return fi local diff diff="$(awk -v a="$readback" -v b="$target_value" 'BEGIN { d = a - b; if (d < 0) d = -d; print d }')" local within_tolerance within_tolerance="$(awk -v d="$diff" -v t="$tolerance" 'BEGIN { print (d <= t) ? "1" : "0" }')" if [ "$within_tolerance" = "1" ]; then pass "$name (chunk is $chunk_size bytes, starts 0x78; wrote $target_value, read back $readback)" else fail "$name (wrote $target_value, read back $readback - outside +-$tolerance tolerance - see $log1 / $log2)" fi } # --- Test 7: stability sweep over bundled sample patches ------------------ # Broad regression net: load every bundled .fxp under extras/sounds/ and # render a held note through it. NOTE: these particular sample files # (10+ years old) load without crashing but - confirmed by diffing # --printParams output with and without --patch - have no effect on the # patch's parameters. This is NOT the effFlagsProgramChunks/usesChunks() # gap an earlier version of this comment claimed (that was wrong - see # tests/README.md's "corrected finding" note; usesChunks() genuinely is # true, and test_chunk_roundtrip above proves the real chunk path works # with files this same build produces). The actual cause is still # unidentified - most likely these specific old files use a compression # or XML-schema shape that JaySynthAudioProcessor::setCurrentProgramState- # Information/getXmlFromBinary no longer parses, not a host-vs-plugin # capability mismatch. Tracked in tools/testhost/TODO.md's Investigate # section. So this test only checks "loading this file doesn't crash the # process", not "loading this file changes the patch" - it still # exercises the malloc/delete[] paths touched by the Memory-management # fixes as a broad smoke test, just not the patch-content-application # path. test_stability_sweep() { local sounds_dir="$REPO_ROOT/extras/sounds" local any=0 while IFS= read -r -d '' fxp; do any=1 local base base="$(basename "$fxp")" local log_file="$WORKDIR/sweep_$(echo "$base" | tr -c 'A-Za-z0-9._-' '_').log" if run_testhost 15 --patch "$fxp" --note 60 100 --duration 0.5 > "$log_file" 2>&1; then if grep -qi "WARNING - output contained NaN" "$log_file"; then fail "stability_sweep: '$base' (NaN/Inf - see $log_file)" else pass "stability_sweep: '$base'" fi else local rc=$? fail "stability_sweep: '$base' (exit code $rc, expected 0 - see $log_file)" fi done < <(find "$sounds_dir" -maxdepth 1 -iname '*.fxp' -print0 2>/dev/null) if [ "$any" -eq 0 ]; then log " (no .fxp files found under $sounds_dir - skipping stability sweep)" fi } # --- Run everything ------------------------------------------------------- log "testhost regression suite" log " testhost: $TESTHOST_BIN" log " plugin: $PLUGIN" log " workdir: $WORKDIR" log "" log "Critical finding #5 - oversized block size:" test_oversized_blocksize log "" log "Critical finding #1 - NRPN controller ID bounds:" test_nrpn_bounds log "" log "Patch/Bank import-export fixes - state round-trip:" test_patch_roundtrip test_bank_roundtrip log "" log "Raw VST2 chunk dispatcher - proves the actual chunk-based path:" test_chunk_roundtrip bank 0.81 test_chunk_roundtrip patch 0.44 log "" log "General stability sweep over bundled sample patches:" test_stability_sweep log "" log "----------------------------------------" log "Results: $PASS_COUNT passed, $FAIL_COUNT failed" if [ "$FAIL_COUNT" -gt 0 ]; then log "Failed:" for n in "${FAILED_NAMES[@]}"; do log " - $n" done log "Artifacts (logs/WAVs) kept in: $WORKDIR" exit 1 fi log "All tests passed. Cleaning up $WORKDIR." rm -rf "$WORKDIR" exit 0