mirror of
https://github.com/sockspls/badfish
synced 2025-04-29 16:23:09 +00:00
a06e7004c1
Since an unknown amount of time the instrumented CI has been a bit flawed, explained here https://github.com/official-stockfish/Stockfish/issues/5185. It also experiences random timeout issues where restarting the workflow fixes it or very long run times (more than other workflows) and is not very portable. The intention of this commit is to port the instrumented.sh to python which also works on other operating systems. It should also be relatively easy for beginners to add new tests to assert stockfish's output and to run it. From the source directory the following command can be run. `python3 ../tests/instrumented.py --none ./stockfish` A test runner will go over the test suites and run the test cases. All instrumented tests should have been ported over. The required python version for this is should be 3.7 (untested) + the requests package, testing.py includes some infrastructure code which setups the testing. fixes https://github.com/official-stockfish/Stockfish/issues/5185 closes https://github.com/official-stockfish/Stockfish/pull/5583 No functional change
520 lines
17 KiB
Python
520 lines
17 KiB
Python
import argparse
|
|
import re
|
|
import sys
|
|
import subprocess
|
|
import pathlib
|
|
import os
|
|
|
|
from testing import (
|
|
EPD,
|
|
TSAN,
|
|
Stockfish as Engine,
|
|
MiniTestFramework,
|
|
OrderedClassMembers,
|
|
Valgrind,
|
|
Syzygy,
|
|
)
|
|
|
|
PATH = pathlib.Path(__file__).parent.resolve()
|
|
CWD = os.getcwd()
|
|
|
|
|
|
def get_prefix():
|
|
if args.valgrind:
|
|
return Valgrind.get_valgrind_command()
|
|
if args.valgrind_thread:
|
|
return Valgrind.get_valgrind_thread_command()
|
|
|
|
return []
|
|
|
|
|
|
def get_threads():
|
|
if args.valgrind_thread or args.sanitizer_thread:
|
|
return 2
|
|
return 1
|
|
|
|
|
|
def get_path():
|
|
return os.path.abspath(os.path.join(CWD, args.stockfish_path))
|
|
|
|
|
|
def postfix_check(output):
|
|
if args.sanitizer_undefined:
|
|
for idx, line in enumerate(output):
|
|
if "runtime error:" in line:
|
|
# print next possible 50 lines
|
|
for i in range(50):
|
|
debug_idx = idx + i
|
|
if debug_idx < len(output):
|
|
print(output[debug_idx])
|
|
return False
|
|
|
|
if args.sanitizer_thread:
|
|
for idx, line in enumerate(output):
|
|
if "WARNING: ThreadSanitizer:" in line:
|
|
# print next possible 50 lines
|
|
for i in range(50):
|
|
debug_idx = idx + i
|
|
if debug_idx < len(output):
|
|
print(output[debug_idx])
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def Stockfish(*args, **kwargs):
|
|
return Engine(get_prefix(), get_path(), *args, **kwargs)
|
|
|
|
|
|
class TestCLI(metaclass=OrderedClassMembers):
|
|
|
|
def beforeAll(self):
|
|
pass
|
|
|
|
def afterAll(self):
|
|
pass
|
|
|
|
def beforeEach(self):
|
|
self.stockfish = None
|
|
|
|
def afterEach(self):
|
|
assert postfix_check(self.stockfish.get_output()) == True
|
|
self.stockfish.clear_output()
|
|
|
|
def test_eval(self):
|
|
self.stockfish = Stockfish("eval".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_nodes_1000(self):
|
|
self.stockfish = Stockfish("go nodes 1000".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_depth_10(self):
|
|
self.stockfish = Stockfish("go depth 10".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_perft_4(self):
|
|
self.stockfish = Stockfish("go perft 4".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_movetime_1000(self):
|
|
self.stockfish = Stockfish("go movetime 1000".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_wtime_8000_btime_8000_winc_500_binc_500(self):
|
|
self.stockfish = Stockfish(
|
|
"go wtime 8000 btime 8000 winc 500 binc 500".split(" "),
|
|
True,
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_wtime_1000_btime_1000_winc_0_binc_0(self):
|
|
self.stockfish = Stockfish(
|
|
"go wtime 1000 btime 1000 winc 0 binc 0".split(" "),
|
|
True,
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_wtime_1000_btime_1000_winc_0_binc_0_movestogo_5(self):
|
|
self.stockfish = Stockfish(
|
|
"go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5".split(" "),
|
|
True,
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_movetime_200(self):
|
|
self.stockfish = Stockfish("go movetime 200".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_go_nodes_20000_searchmoves_e2e4_d2d4(self):
|
|
self.stockfish = Stockfish(
|
|
"go nodes 20000 searchmoves e2e4 d2d4".split(" "), True
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_bench_128_threads_8_default_depth(self):
|
|
self.stockfish = Stockfish(
|
|
f"bench 128 {get_threads()} 8 default depth".split(" "),
|
|
True,
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_bench_128_threads_3_bench_tmp_epd_depth(self):
|
|
self.stockfish = Stockfish(
|
|
f"bench 128 {get_threads()} 3 {os.path.join(PATH,'bench_tmp.epd')} depth".split(
|
|
" "
|
|
),
|
|
True,
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_d(self):
|
|
self.stockfish = Stockfish("d".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_compiler(self):
|
|
self.stockfish = Stockfish("compiler".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_license(self):
|
|
self.stockfish = Stockfish("license".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_uci(self):
|
|
self.stockfish = Stockfish("uci".split(" "), True)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
def test_export_net_verify_nnue(self):
|
|
current_path = os.path.abspath(os.getcwd())
|
|
self.stockfish = Stockfish(
|
|
f"export_net {os.path.join(current_path , 'verify.nnue')}".split(" "), True
|
|
)
|
|
assert self.stockfish.process.returncode == 0
|
|
|
|
# verify the generated net equals the base net
|
|
|
|
def test_network_equals_base(self):
|
|
self.stockfish = Stockfish(
|
|
["uci"],
|
|
True,
|
|
)
|
|
|
|
output = self.stockfish.process.stdout
|
|
|
|
# find line
|
|
for line in output.split("\n"):
|
|
if "option name EvalFile type string default" in line:
|
|
network = line.split(" ")[-1]
|
|
break
|
|
|
|
# find network file in src dir
|
|
network = os.path.join(PATH.parent.resolve(), "src", network)
|
|
|
|
if not os.path.exists(network):
|
|
print(
|
|
f"Network file {network} not found, please download the network file over the make command."
|
|
)
|
|
assert False
|
|
|
|
diff = subprocess.run(["diff", network, f"verify.nnue"])
|
|
|
|
assert diff.returncode == 0
|
|
|
|
|
|
class TestInteractive(metaclass=OrderedClassMembers):
|
|
def beforeAll(self):
|
|
self.stockfish = Stockfish()
|
|
|
|
def afterAll(self):
|
|
self.stockfish.quit()
|
|
assert self.stockfish.close() == 0
|
|
|
|
def afterEach(self):
|
|
assert postfix_check(self.stockfish.get_output()) == True
|
|
self.stockfish.clear_output()
|
|
|
|
def test_startup_output(self):
|
|
self.stockfish.starts_with("Stockfish")
|
|
|
|
def test_uci_command(self):
|
|
self.stockfish.send_command("uci")
|
|
self.stockfish.equals("uciok")
|
|
|
|
def test_set_threads_option(self):
|
|
self.stockfish.send_command(f"setoption name Threads value {get_threads()}")
|
|
|
|
def test_ucinewgame_and_startpos_nodes_1000(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go nodes 1000")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_ucinewgame_and_startpos_moves(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position startpos moves e2e4 e7e6")
|
|
self.stockfish.send_command("go nodes 1000")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_1(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1")
|
|
self.stockfish.send_command("go nodes 1000")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_2_flip(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1")
|
|
self.stockfish.send_command("flip")
|
|
self.stockfish.send_command("go nodes 1000")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_depth_5_with_callback(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go depth 5")
|
|
|
|
def callback(output):
|
|
regex = r"info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv"
|
|
if output.startswith("info depth") and not re.match(regex, output):
|
|
assert False
|
|
if output.startswith("bestmove"):
|
|
return True
|
|
return False
|
|
|
|
self.stockfish.check_output(callback)
|
|
|
|
def test_ucinewgame_and_go_depth_9(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("setoption name UCI_ShowWDL value true")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go depth 9")
|
|
|
|
depth = 1
|
|
|
|
def callback(output):
|
|
nonlocal depth
|
|
|
|
regex = rf"info depth {depth} seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv"
|
|
|
|
if output.startswith("info depth"):
|
|
if not re.match(regex, output):
|
|
assert False
|
|
depth += 1
|
|
|
|
if output.startswith("bestmove"):
|
|
assert depth == 10
|
|
return True
|
|
|
|
return False
|
|
|
|
self.stockfish.check_output(callback)
|
|
|
|
def test_clear_hash(self):
|
|
self.stockfish.send_command("setoption name Clear Hash")
|
|
|
|
def test_fen_position_mate_1(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6"
|
|
)
|
|
self.stockfish.send_command("go depth 18")
|
|
|
|
self.stockfish.expect("* score mate 1 * pv d5e6")
|
|
self.stockfish.equals("bestmove d5e6")
|
|
|
|
def test_fen_position_mate_minus_1(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -"
|
|
)
|
|
self.stockfish.send_command("go depth 18")
|
|
self.stockfish.expect("* score mate -1 *")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_fixed_node(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1"
|
|
)
|
|
self.stockfish.send_command("go nodes 500000")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_with_mate_go_depth(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
|
|
)
|
|
self.stockfish.send_command("go depth 18 searchmoves c6d7")
|
|
self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
|
|
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_with_mate_go_mate(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
|
|
)
|
|
self.stockfish.send_command("go mate 2 searchmoves c6d7")
|
|
self.stockfish.expect("* score mate 2 * pv c6d7 *")
|
|
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_with_mate_go_nodes(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
|
|
)
|
|
self.stockfish.send_command("go nodes 500000 searchmoves c6d7")
|
|
self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
|
|
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_depth_27(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - -"
|
|
)
|
|
self.stockfish.send_command("go depth 27")
|
|
self.stockfish.contains("score mate -2")
|
|
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_with_mate_go_depth_and_promotion(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q"
|
|
)
|
|
self.stockfish.send_command("go depth 18")
|
|
self.stockfish.expect("* score mate 1 * pv f7f5")
|
|
self.stockfish.starts_with("bestmove f7f5")
|
|
|
|
def test_fen_position_with_mate_go_depth_and_searchmoves(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
|
|
)
|
|
self.stockfish.send_command("go depth 18 searchmoves c6d7")
|
|
self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
|
|
|
|
self.stockfish.starts_with("bestmove c6d7")
|
|
|
|
def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command(
|
|
"position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7"
|
|
)
|
|
self.stockfish.send_command("go depth 18 searchmoves e3e2")
|
|
self.stockfish.expect("* score mate -1 * pv e3e2 f7f5")
|
|
self.stockfish.starts_with("bestmove e3e2")
|
|
|
|
def test_verify_nnue_network(self):
|
|
current_path = os.path.abspath(os.getcwd())
|
|
Stockfish(
|
|
f"export_net {os.path.join(current_path , 'verify.nnue')}".split(" "), True
|
|
)
|
|
|
|
self.stockfish.send_command("setoption name EvalFile value verify.nnue")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go depth 5")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_multipv_setting(self):
|
|
self.stockfish.send_command("setoption name MultiPV value 4")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go depth 5")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
def test_fen_position_with_skill_level(self):
|
|
self.stockfish.send_command("setoption name Skill Level value 10")
|
|
self.stockfish.send_command("position startpos")
|
|
self.stockfish.send_command("go depth 5")
|
|
self.stockfish.starts_with("bestmove")
|
|
|
|
self.stockfish.send_command("setoption name Skill Level value 20")
|
|
|
|
|
|
class TestSyzygy(metaclass=OrderedClassMembers):
|
|
def beforeAll(self):
|
|
self.stockfish = Stockfish()
|
|
|
|
def afterAll(self):
|
|
self.stockfish.quit()
|
|
assert self.stockfish.close() == 0
|
|
|
|
def afterEach(self):
|
|
assert postfix_check(self.stockfish.get_output()) == True
|
|
self.stockfish.clear_output()
|
|
|
|
def test_syzygy_setup(self):
|
|
self.stockfish.starts_with("Stockfish")
|
|
self.stockfish.send_command("uci")
|
|
self.stockfish.send_command(
|
|
f"setoption name SyzygyPath value {os.path.join(PATH, 'syzygy')}"
|
|
)
|
|
self.stockfish.expect(
|
|
"info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
|
|
)
|
|
|
|
def test_syzygy_bench(self):
|
|
self.stockfish.send_command("bench 128 1 8 default depth")
|
|
self.stockfish.expect("Nodes searched :*")
|
|
|
|
def test_syzygy_position(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1")
|
|
self.stockfish.send_command("go depth 5")
|
|
|
|
def check_output(output):
|
|
if "score cp 20000" in output or "score mate" in output:
|
|
return True
|
|
|
|
self.stockfish.check_output(check_output)
|
|
self.stockfish.expect("bestmove *")
|
|
|
|
def test_syzygy_position_2(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1")
|
|
self.stockfish.send_command("go depth 5")
|
|
|
|
def check_output(output):
|
|
if "score cp 20000" in output or "score mate" in output:
|
|
return True
|
|
|
|
self.stockfish.check_output(check_output)
|
|
self.stockfish.expect("bestmove *")
|
|
|
|
def test_syzygy_position_3(self):
|
|
self.stockfish.send_command("ucinewgame")
|
|
self.stockfish.send_command("position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1")
|
|
self.stockfish.send_command("go depth 5")
|
|
|
|
def check_output(output):
|
|
if "score cp -20000" in output or "score mate" in output:
|
|
return True
|
|
|
|
self.stockfish.check_output(check_output)
|
|
self.stockfish.expect("bestmove *")
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description="Run Stockfish with testing options")
|
|
parser.add_argument("--valgrind", action="store_true", help="Run valgrind testing")
|
|
parser.add_argument(
|
|
"--valgrind-thread", action="store_true", help="Run valgrind-thread testing"
|
|
)
|
|
parser.add_argument(
|
|
"--sanitizer-undefined",
|
|
action="store_true",
|
|
help="Run sanitizer-undefined testing",
|
|
)
|
|
parser.add_argument(
|
|
"--sanitizer-thread", action="store_true", help="Run sanitizer-thread testing"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--none", action="store_true", help="Run without any testing options"
|
|
)
|
|
parser.add_argument("stockfish_path", type=str, help="Path to Stockfish binary")
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = parse_args()
|
|
|
|
EPD.create_bench_epd()
|
|
TSAN.set_tsan_option()
|
|
Syzygy.download_syzygy()
|
|
|
|
framework = MiniTestFramework()
|
|
|
|
# Each test suite will be ran inside a temporary directory
|
|
framework.run([TestCLI, TestInteractive, TestSyzygy])
|
|
|
|
EPD.delete_bench_epd()
|
|
TSAN.unset_tsan_option()
|
|
|
|
if framework.has_failed():
|
|
sys.exit(1)
|
|
|
|
sys.exit(0)
|