mirror of
https://github.com/sockspls/badfish
synced 2025-04-30 00:33:09 +00:00
Improve CI
the recent refactoring has shown some limitations of our testing, hence we add a couple of more tests including: * expected mate score * expected mated score * expected in TB win score * expected in TB loss score * expected info line output * expected info line output (wdl) closes https://github.com/official-stockfish/Stockfish/pull/5181 No functional change
This commit is contained in:
Joost VandeVondele
parent
6fc7da44ad
commit
1a8de45b8c
2 changed files with 112 additions and 7 deletions
3
.github/workflows/sanitizers.yml
vendored
3
.github/workflows/sanitizers.yml
vendored
|
|
@ -31,6 +31,9 @@ jobs:
|
|||
- name: Run under valgrind-thread
|
||||
make_option: ""
|
||||
instrumented_option: valgrind-thread
|
||||
- name: Run non-instrumented
|
||||
make_option: ""
|
||||
instrumented_option: none
|
||||
defaults:
|
||||
run:
|
||||
working-directory: src
|
||||
|
|
|
|||
|
|
@ -21,14 +21,14 @@ case $1 in
|
|||
echo "valgrind testing started"
|
||||
prefix=''
|
||||
exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
|
||||
postfix='1>/dev/null'
|
||||
postfix=''
|
||||
threads="1"
|
||||
;;
|
||||
--valgrind-thread)
|
||||
echo "valgrind-thread testing started"
|
||||
prefix=''
|
||||
exeprefix='valgrind --fair-sched=try --error-exitcode=42'
|
||||
postfix='1>/dev/null'
|
||||
postfix=''
|
||||
threads="2"
|
||||
;;
|
||||
--sanitizer-undefined)
|
||||
|
|
@ -112,7 +112,12 @@ diff $network verify.nnue
|
|||
# more general testing, following an uci protocol exchange
|
||||
cat << EOF > game.exp
|
||||
set timeout 240
|
||||
# to correctly catch eof we need the following line
|
||||
# expect_before timeout { exit 2 } eof { exit 3 }
|
||||
expect_before timeout { exit 2 }
|
||||
|
||||
spawn $exeprefix ./stockfish
|
||||
expect "Stockfish"
|
||||
|
||||
send "uci\n"
|
||||
expect "uciok"
|
||||
|
|
@ -125,27 +130,101 @@ cat << EOF > game.exp
|
|||
send "go nodes 1000\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position startpos moves e2e4 e7e6\n"
|
||||
send "go nodes 1000\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
|
||||
send "go depth 10\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "setoption name UCI_ShowWDL value true\n"
|
||||
send "position startpos\n"
|
||||
send "ucinewgame\n"
|
||||
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
|
||||
send "flip\n"
|
||||
send "go depth 5\n"
|
||||
send "go depth 10\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "setoption name Skill Level value 10\n"
|
||||
send "ucinewgame\n"
|
||||
send "position startpos\n"
|
||||
send "go depth 5\n"
|
||||
expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect "bestmove"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "setoption name UCI_ShowWDL value true\n"
|
||||
send "position startpos\n"
|
||||
send "go depth 9\n"
|
||||
expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
|
||||
expect "bestmove"
|
||||
|
||||
send "setoption name Clear Hash\n"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
|
||||
send "go depth 18\n"
|
||||
expect "score mate 1"
|
||||
expect "pv d5e6"
|
||||
expect "bestmove d5e6"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
|
||||
send "go depth 18\n"
|
||||
expect "score mate -1"
|
||||
expect "bestmove"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
||||
send "go depth 18\n"
|
||||
expect "score mate 2 * pv c6d7 * f7f5"
|
||||
expect "bestmove c6d7"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
||||
send "go mate 2\n"
|
||||
expect "score mate 2 * pv c6d7"
|
||||
expect "bestmove c6d7"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
||||
send "go nodes 10000\n"
|
||||
expect "score mate 2 * pv c6d7 * f7f5"
|
||||
expect "bestmove c6d7"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
|
||||
send "go depth 18\n"
|
||||
expect "score mate -2"
|
||||
expect "pv d5e6 c8d8"
|
||||
expect "bestmove d5e6"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
|
||||
send "go depth 18\n"
|
||||
expect "score mate 1 * pv f7f5"
|
||||
expect "bestmove f7f5"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
|
||||
send "go depth 18 searchmoves c6d7\n"
|
||||
expect "score mate 2 * pv c6d7 * f7f5"
|
||||
expect "bestmove c6d7"
|
||||
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
|
||||
send "go depth 18 searchmoves e3e2\n"
|
||||
expect "score mate -1 * pv e3e2 f7f5"
|
||||
expect "bestmove e3e2"
|
||||
|
||||
send "setoption name EvalFile value verify.nnue\n"
|
||||
send "position startpos\n"
|
||||
send "go depth 5\n"
|
||||
|
|
@ -154,6 +233,13 @@ cat << EOF > game.exp
|
|||
send "setoption name MultiPV value 4\n"
|
||||
send "position startpos\n"
|
||||
send "go depth 5\n"
|
||||
expect "bestmove"
|
||||
|
||||
send "setoption name Skill Level value 10\n"
|
||||
send "position startpos\n"
|
||||
send "go depth 5\n"
|
||||
expect "bestmove"
|
||||
send "setoption name Skill Level value 20\n"
|
||||
|
||||
send "quit\n"
|
||||
expect eof
|
||||
|
|
@ -171,17 +257,30 @@ fi
|
|||
|
||||
cat << EOF > syzygy.exp
|
||||
set timeout 240
|
||||
# to correctly catch eof we need the following line
|
||||
# expect_before timeout { exit 2 } eof { exit 3 }
|
||||
expect_before timeout { exit 2 }
|
||||
spawn $exeprefix ./stockfish
|
||||
expect "Stockfish"
|
||||
send "uci\n"
|
||||
send "setoption name SyzygyPath value ../tests/syzygy/\n"
|
||||
expect "info string Found 35 tablebases" {} timeout {exit 1}
|
||||
expect "info string Found 35 tablebases"
|
||||
send "bench 128 1 8 default depth\n"
|
||||
expect "Nodes searched :"
|
||||
send "ucinewgame\n"
|
||||
send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
|
||||
send "go depth 5\n"
|
||||
expect -re {score cp 20000|score mate}
|
||||
expect "bestmove"
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
|
||||
send "go depth 5\n"
|
||||
expect -re {score cp 20000|score mate}
|
||||
expect "bestmove"
|
||||
send "ucinewgame\n"
|
||||
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
|
||||
send "go depth 5\n"
|
||||
expect -re {score cp -20000|score mate}
|
||||
expect "bestmove"
|
||||
send "quit\n"
|
||||
expect eof
|
||||
|
|
@ -194,6 +293,9 @@ EOF
|
|||
for exp in game.exp syzygy.exp
|
||||
do
|
||||
|
||||
echo "======== $exp =============="
|
||||
cat $exp
|
||||
echo "============================"
|
||||
echo "$prefix expect $exp $postfix"
|
||||
eval "$prefix expect $exp $postfix"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue