mirror of
https://github.com/sockspls/badfish
synced 2025-04-29 16:23:09 +00:00
ab00c24c7e
due to https://github.com/official-stockfish/Stockfish/issues/5185 some CI tests are skipped. This patch fixes a few tests that need updating. closes https://github.com/official-stockfish/Stockfish/pull/5560 No functional change
301 lines
8.8 KiB
Bash
Executable file
301 lines
8.8 KiB
Bash
Executable file
#!/bin/bash
|
|
# check for errors under Valgrind or sanitizers.
|
|
|
|
error()
|
|
{
|
|
echo "instrumented testing failed on line $1"
|
|
exit 1
|
|
}
|
|
trap 'error ${LINENO}' ERR
|
|
|
|
# define suitable post and prefixes for testing options
|
|
case $1 in
|
|
--valgrind)
|
|
echo "valgrind testing started"
|
|
prefix=''
|
|
exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
|
|
postfix=''
|
|
threads="1"
|
|
;;
|
|
--valgrind-thread)
|
|
echo "valgrind-thread testing started"
|
|
prefix=''
|
|
exeprefix='valgrind --fair-sched=try --error-exitcode=42'
|
|
postfix=''
|
|
threads="2"
|
|
;;
|
|
--sanitizer-undefined)
|
|
echo "sanitizer-undefined testing started"
|
|
prefix='!'
|
|
exeprefix=''
|
|
postfix='2>&1 | grep -A50 "runtime error:"'
|
|
threads="1"
|
|
;;
|
|
--sanitizer-thread)
|
|
echo "sanitizer-thread testing started"
|
|
prefix='!'
|
|
exeprefix=''
|
|
postfix='2>&1 | grep -A50 "WARNING: ThreadSanitizer:"'
|
|
threads="2"
|
|
|
|
cat << EOF > tsan.supp
|
|
race:Stockfish::TTEntry::read
|
|
race:Stockfish::TTEntry::save
|
|
|
|
race:Stockfish::TranspositionTable::probe
|
|
race:Stockfish::TranspositionTable::hashfull
|
|
|
|
EOF
|
|
|
|
export TSAN_OPTIONS="suppressions=./tsan.supp"
|
|
|
|
;;
|
|
*)
|
|
echo "unknown testing started"
|
|
prefix=''
|
|
exeprefix=''
|
|
postfix=''
|
|
threads="1"
|
|
;;
|
|
esac
|
|
|
|
cat << EOF > bench_tmp.epd
|
|
Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
|
|
rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
|
|
3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
|
|
r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
|
|
EOF
|
|
|
|
# simple command line testing
|
|
for args in "eval" \
|
|
"go nodes 1000" \
|
|
"go depth 10" \
|
|
"go perft 4" \
|
|
"go movetime 1000" \
|
|
"go wtime 8000 btime 8000 winc 500 binc 500" \
|
|
"go wtime 1000 btime 1000 winc 0 binc 0" \
|
|
"go wtime 1000 btime 1000 winc 0 binc 0" \
|
|
"go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5" \
|
|
"go movetime 200" \
|
|
"go nodes 20000 searchmoves e2e4 d2d4" \
|
|
"bench 128 $threads 8 default depth" \
|
|
"bench 128 $threads 3 bench_tmp.epd depth" \
|
|
"export_net verify.nnue" \
|
|
"d" \
|
|
"compiler" \
|
|
"license" \
|
|
"uci"
|
|
do
|
|
|
|
echo "$prefix $exeprefix ./stockfish $args $postfix"
|
|
eval "$prefix $exeprefix ./stockfish $args $postfix"
|
|
|
|
done
|
|
|
|
# verify the generated net equals the base net
|
|
network=`./stockfish uci | grep 'option name EvalFile type string default' | awk '{print $NF}'`
|
|
echo "Comparing $network to the written verify.nnue"
|
|
diff $network verify.nnue
|
|
|
|
# more general testing, following an uci protocol exchange
|
|
cat << EOF > game.exp
|
|
set timeout 240
|
|
# to correctly catch eof we need the following line
|
|
# expect_before timeout { exit 2 } eof { exit 3 }
|
|
expect_before timeout { exit 2 }
|
|
|
|
spawn $exeprefix ./stockfish
|
|
expect "Stockfish"
|
|
|
|
send "uci\n"
|
|
expect "uciok"
|
|
|
|
# send "setoption name Debug Log File value debug.log\n"
|
|
send "setoption name Threads value $threads\n"
|
|
|
|
send "ucinewgame\n"
|
|
send "position startpos\n"
|
|
send "go nodes 1000\n"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position startpos moves e2e4 e7e6\n"
|
|
send "go nodes 1000\n"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
|
|
send "go depth 10\n"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
|
|
send "flip\n"
|
|
send "go depth 10\n"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position startpos\n"
|
|
send "go depth 5\n"
|
|
expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "setoption name UCI_ShowWDL value true\n"
|
|
send "position startpos\n"
|
|
send "go depth 9\n"
|
|
expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
|
expect "bestmove"
|
|
|
|
send "setoption name Clear Hash\n"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
|
|
send "go depth 18\n"
|
|
expect "score mate 1"
|
|
expect "pv d5e6"
|
|
expect "bestmove d5e6"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
|
|
send "go depth 18\n"
|
|
expect "score mate -1"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 7K/P1p1p1p1/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1\n"
|
|
send "go nodes 500000\n"
|
|
expect "bestmove"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
|
send "go depth 18 searchmoves c6d7\n"
|
|
expect "score mate 2 * pv c6d7 * f7f5"
|
|
expect "bestmove c6d7"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
|
send "go mate 2 searchmoves c6d7\n"
|
|
expect "score mate 2 * pv c6d7"
|
|
expect "bestmove c6d7"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
|
send "go nodes 500000 searchmoves c6d7\n"
|
|
expect "score mate 2 * pv c6d7 * f7f5"
|
|
expect "bestmove c6d7"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
|
|
send "go depth 27\n"
|
|
expect "score mate -2"
|
|
expect "pv d5e6 c8d8"
|
|
expect "bestmove d5e6"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
|
|
send "go depth 18\n"
|
|
expect "score mate 1 * pv f7f5"
|
|
expect "bestmove f7f5"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
|
send "go depth 18 searchmoves c6d7\n"
|
|
expect "score mate 2 * pv c6d7 * f7f5"
|
|
expect "bestmove c6d7"
|
|
|
|
send "ucinewgame\n"
|
|
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
|
|
send "go depth 18 searchmoves e3e2\n"
|
|
expect "score mate -1 * pv e3e2 f7f5"
|
|
expect "bestmove e3e2"
|
|
|
|
send "setoption name EvalFile value verify.nnue\n"
|
|
send "position startpos\n"
|
|
send "go depth 5\n"
|
|
expect "bestmove"
|
|
|
|
send "setoption name MultiPV value 4\n"
|
|
send "position startpos\n"
|
|
send "go depth 5\n"
|
|
expect "bestmove"
|
|
|
|
send "setoption name Skill Level value 10\n"
|
|
send "position startpos\n"
|
|
send "go depth 5\n"
|
|
expect "bestmove"
|
|
send "setoption name Skill Level value 20\n"
|
|
|
|
send "quit\n"
|
|
expect eof
|
|
|
|
# return error code of the spawned program, useful for Valgrind
|
|
lassign [wait] pid spawnid os_error_flag value
|
|
exit \$value
|
|
EOF
|
|
|
|
#download TB as needed
|
|
if [ ! -d ../tests/syzygy ]; then
|
|
curl -sL https://api.github.com/repos/niklasf/python-chess/tarball/9b9aa13f9f36d08aadfabff872882f4ab1494e95 | tar -xzf -
|
|
mv niklasf-python-chess-9b9aa13 ../tests/syzygy
|
|
fi
|
|
|
|
cat << EOF > syzygy.exp
|
|
set timeout 240
|
|
# to correctly catch eof we need the following line
|
|
# expect_before timeout { exit 2 } eof { exit 3 }
|
|
expect_before timeout { exit 2 }
|
|
spawn $exeprefix ./stockfish
|
|
expect "Stockfish"
|
|
send "uci\n"
|
|
send "setoption name SyzygyPath value ../tests/syzygy/\n"
|
|
expect "info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
|
|
send "bench 128 1 8 default depth\n"
|
|
expect "Nodes searched :"
|
|
send "ucinewgame\n"
|
|
send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
|
|
send "go depth 5\n"
|
|
expect -re {score cp 20000|score mate}
|
|
expect "bestmove"
|
|
send "ucinewgame\n"
|
|
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
|
|
send "go depth 5\n"
|
|
expect -re {score cp 20000|score mate}
|
|
expect "bestmove"
|
|
send "ucinewgame\n"
|
|
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
|
|
send "go depth 5\n"
|
|
expect -re {score cp -20000|score mate}
|
|
expect "bestmove"
|
|
send "quit\n"
|
|
expect eof
|
|
|
|
# return error code of the spawned program, useful for Valgrind
|
|
lassign [wait] pid spawnid os_error_flag value
|
|
exit \$value
|
|
EOF
|
|
|
|
for exp in game.exp syzygy.exp
|
|
do
|
|
|
|
echo "======== $exp =============="
|
|
cat $exp
|
|
echo "============================"
|
|
echo "$prefix expect $exp $postfix"
|
|
eval "$prefix expect $exp $postfix"
|
|
|
|
rm $exp
|
|
|
|
done
|
|
|
|
rm -f tsan.supp bench_tmp.epd
|
|
|
|
echo "instrumented testing OK"
|