1
0
Fork 0
mirror of https://github.com/sockspls/badfish synced 2025-04-29 16:23:09 +00:00

Improve CI

the recent refactoring has shown some limitations of our testing, hence we add a couple of more tests including:
* expected mate score
* expected mated score
* expected in TB win score
* expected in TB loss score
* expected info line output
* expected info line output (wdl)

closes https://github.com/official-stockfish/Stockfish/pull/5181

No functional change
This commit is contained in:
Joost VandeVondele 2024-04-20 15:33:07 +02:00
parent 6fc7da44ad
commit 1a8de45b8c
2 changed files with 112 additions and 7 deletions

View file

@ -31,6 +31,9 @@ jobs:
- name: Run under valgrind-thread
make_option: ""
instrumented_option: valgrind-thread
- name: Run non-instrumented
make_option: ""
instrumented_option: none
defaults:
run:
working-directory: src

View file

@ -21,14 +21,14 @@ case $1 in
echo "valgrind testing started"
prefix=''
exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
postfix='1>/dev/null'
postfix=''
threads="1"
;;
--valgrind-thread)
echo "valgrind-thread testing started"
prefix=''
exeprefix='valgrind --fair-sched=try --error-exitcode=42'
postfix='1>/dev/null'
postfix=''
threads="2"
;;
--sanitizer-undefined)
@ -112,7 +112,12 @@ diff $network verify.nnue
# more general testing, following an uci protocol exchange
cat << EOF > game.exp
set timeout 240
# to correctly catch eof we need the following line
# expect_before timeout { exit 2 } eof { exit 3 }
expect_before timeout { exit 2 }
spawn $exeprefix ./stockfish
expect "Stockfish"
send "uci\n"
expect "uciok"
@ -125,27 +130,101 @@ cat << EOF > game.exp
send "go nodes 1000\n"
expect "bestmove"
send "ucinewgame\n"
send "position startpos moves e2e4 e7e6\n"
send "go nodes 1000\n"
expect "bestmove"
send "ucinewgame\n"
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
send "go depth 10\n"
expect "bestmove"
send "setoption name UCI_ShowWDL value true\n"
send "position startpos\n"
send "ucinewgame\n"
send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
send "flip\n"
send "go depth 5\n"
send "go depth 10\n"
expect "bestmove"
send "setoption name Skill Level value 10\n"
send "ucinewgame\n"
send "position startpos\n"
send "go depth 5\n"
expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect "bestmove"
send "ucinewgame\n"
send "setoption name UCI_ShowWDL value true\n"
send "position startpos\n"
send "go depth 9\n"
expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
expect "bestmove"
send "setoption name Clear Hash\n"
send "ucinewgame\n"
send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
send "go depth 18\n"
expect "score mate 1"
expect "pv d5e6"
expect "bestmove d5e6"
send "ucinewgame\n"
send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
send "go depth 18\n"
expect "score mate -1"
expect "bestmove"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
send "go depth 18\n"
expect "score mate 2 * pv c6d7 * f7f5"
expect "bestmove c6d7"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
send "go mate 2\n"
expect "score mate 2 * pv c6d7"
expect "bestmove c6d7"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
send "go nodes 10000\n"
expect "score mate 2 * pv c6d7 * f7f5"
expect "bestmove c6d7"
send "ucinewgame\n"
send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
send "go depth 18\n"
expect "score mate -2"
expect "pv d5e6 c8d8"
expect "bestmove d5e6"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
send "go depth 18\n"
expect "score mate 1 * pv f7f5"
expect "bestmove f7f5"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
send "go depth 18 searchmoves c6d7\n"
expect "score mate 2 * pv c6d7 * f7f5"
expect "bestmove c6d7"
send "ucinewgame\n"
send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
send "go depth 18 searchmoves e3e2\n"
expect "score mate -1 * pv e3e2 f7f5"
expect "bestmove e3e2"
send "setoption name EvalFile value verify.nnue\n"
send "position startpos\n"
send "go depth 5\n"
@ -154,6 +233,13 @@ cat << EOF > game.exp
send "setoption name MultiPV value 4\n"
send "position startpos\n"
send "go depth 5\n"
expect "bestmove"
send "setoption name Skill Level value 10\n"
send "position startpos\n"
send "go depth 5\n"
expect "bestmove"
send "setoption name Skill Level value 20\n"
send "quit\n"
expect eof
@ -171,17 +257,30 @@ fi
cat << EOF > syzygy.exp
set timeout 240
# to correctly catch eof we need the following line
# expect_before timeout { exit 2 } eof { exit 3 }
expect_before timeout { exit 2 }
spawn $exeprefix ./stockfish
expect "Stockfish"
send "uci\n"
send "setoption name SyzygyPath value ../tests/syzygy/\n"
expect "info string Found 35 tablebases" {} timeout {exit 1}
expect "info string Found 35 tablebases"
send "bench 128 1 8 default depth\n"
expect "Nodes searched :"
send "ucinewgame\n"
send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
send "go depth 5\n"
expect -re {score cp 20000|score mate}
expect "bestmove"
send "ucinewgame\n"
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
send "go depth 5\n"
expect -re {score cp 20000|score mate}
expect "bestmove"
send "ucinewgame\n"
send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
send "go depth 5\n"
expect -re {score cp -20000|score mate}
expect "bestmove"
send "quit\n"
expect eof
@ -194,6 +293,9 @@ EOF
for exp in game.exp syzygy.exp
do
echo "======== $exp =============="
cat $exp
echo "============================"
echo "$prefix expect $exp $postfix"
eval "$prefix expect $exp $postfix"