Rewrite benchmark

First step in improving bench to handle
arbitrary UCI commands so to test many
more code paths.

This first patch just set the new code
structure.

No functional change.

src/benchmark.cpp

@@ -23,10 +23,7 @@
 #include <istream>
 #include <vector>
 
-#include "misc.h"
 #include "position.h"
-#include "search.h"
-#include "thread.h"
 #include "uci.h"
 
 using namespace std;
@@ -88,19 +85,17 @@ const vector<string> Defaults = {
 
 } // namespace
 
-/// benchmark() runs a simple benchmark by letting Stockfish analyze a set
+/// setup_bench() builds a list of UCI commands to be run by bench. There
-/// of positions for a given limit each. There are five parameters: the
+/// are five parameters: TT size, number of search threads that should
-/// transposition table size, the number of search threads that should
 /// be used, the limit value spent for each position (optional, default is
 /// depth 13), an optional file name where to look for positions in FEN
 /// format (defaults are the positions defined above) and the type of the
 /// limit value: depth (default), time in millisecs or number of nodes.
 
-void benchmark(const Position& current, istream& is) {
+std::vector<string> setup_bench(const Position& current , istream& is) {
 
-  string token;
+  vector<string> fens, list;
-  vector<string> fens;
+  string go, token;
-  Search::LimitsType limits;
 
   // Assign default values to missing arguments
   string ttSize    = (is >> token) ? token : "16";
@@ -109,22 +104,10 @@ void benchmark(const Position& current, istream& is) {
   string fenFile   = (is >> token) ? token : "default";
   string limitType = (is >> token) ? token : "depth";
 
-  Search::clear(); // Wait for search finished
+  // Build 'go' string (movetime is in millisecs)
-  Options["Threads"] = threads;
+  go = "go " + limitType + " " + limit;
-  Options["Hash"]    = ttSize;
-
-  if (limitType == "time")
-      limits.movetime = stoi(limit); // movetime is in millisecs
-
-  else if (limitType == "nodes")
-      limits.nodes = stoll(limit);
-
-  else if (limitType == "mate")
-      limits.mate = stoi(limit);
-
-  else
-      limits.depth = stoi(limit);
 
+  // Get test positions fens
   if (fenFile == "default")
       fens = Defaults;
 
@@ -139,7 +122,7 @@ void benchmark(const Position& current, istream& is) {
       if (!file.is_open())
       {
           cerr << "Unable to open file " << fenFile << endl;
-          return;
+          exit(EXIT_FAILURE);
       }
 
       while (getline(file, fen))
@@ -149,35 +132,16 @@ void benchmark(const Position& current, istream& is) {
       file.close();
   }
 
-  uint64_t nodes = 0;
+  // Build UCI command list
-  TimePoint elapsed = now();
+  list.emplace_back("ucinewgame");
-  Position pos;
+  list.emplace_back("setoption name Threads value " + threads);
+  list.emplace_back("setoption name Hash value " + ttSize);
 
-  for (size_t i = 0; i < fens.size(); ++i)
+  for (const string& fen : fens)
   {
-      StateListPtr states(new std::deque<StateInfo>(1));
+      list.emplace_back("position fen " + fen);
-      pos.set(fens[i], Options["UCI_Chess960"], &states->back(), Threads.main());
+      list.emplace_back(go);
-
-      cerr << "\nPosition: " << i + 1 << '/' << fens.size() << endl;
-
-      if (limitType == "perft")
-          nodes += Search::perft(pos, limits.depth * ONE_PLY);
-
-      else
-      {
-          limits.startTime = now();
-          Threads.start_thinking(pos, states, limits);
-          Threads.main()->wait_for_search_finished();
-          nodes += Threads.nodes_searched();
-      }
   }
 
-  elapsed = now() - elapsed + 1; // Ensure positivity to avoid a 'divide by zero'
+  return list;
-
-  dbg_print(); // Just before exiting
-
-  cerr << "\n==========================="
-       << "\nTotal time (ms) : " << elapsed
-       << "\nNodes searched  : " << nodes
-       << "\nNodes/second    : " << 1000 * nodes / elapsed << endl;
 }

src/uci.cpp

@@ -35,7 +35,7 @@
 
 using namespace std;
 
-extern void benchmark(const Position& pos, istream& is);
+extern std::vector<string> setup_bench(const Position&, istream&);
 
 namespace {
 
@@ -134,6 +134,48 @@ namespace {
     Threads.start_thinking(pos, states, limits, ponderMode);
   }
 
+
+  // bench() is called when engine receives the "bench" command. Firstly
+  // a list of UCI commands is setup according to bench parameters, then
+  // it is run one by one printing summaries at the end.
+
+  void bench(Position& pos, istream& args, StateListPtr& states) {
+
+    string token;
+    uint64_t num, nodes = 0, cnt = 1;
+
+    vector<string> list = setup_bench(pos, args);
+    num = count_if (list.begin(), list.end(), [](string s) { return s.find("go ") == 0; });
+
+    TimePoint elapsed = now();
+
+    for (const auto& cmd : list)
+    {
+        istringstream is(cmd);
+        is >> skipws >> token;
+
+        if (token == "go")
+        {
+            cerr << "\nPosition: " << cnt++ << '/' << num << endl;
+            go(pos, is, states);
+            Threads.main()->wait_for_search_finished();
+            nodes += Threads.nodes_searched();
+        }
+        else if (token == "setoption")  setoption(is);
+        else if (token == "position")   position(pos, is, states);
+        else if (token == "ucinewgame") Search::clear();
+    }
+
+    elapsed = now() - elapsed + 1; // Ensure positivity to avoid a 'divide by zero'
+
+    dbg_print(); // Just before exiting
+
+    cerr << "\n==========================="
+         << "\nTotal time (ms) : " << elapsed
+         << "\nNodes searched  : " << nodes
+         << "\nNodes/second    : " << 1000 * nodes / elapsed << endl;
+  }
+
 } // namespace
 
 
@@ -190,7 +232,7 @@ void UCI::loop(int argc, char* argv[]) {
 
       // Additional custom non-UCI commands, mainly for debugging
       else if (token == "flip")  pos.flip();
-      else if (token == "bench") benchmark(pos, is);
+      else if (token == "bench") bench(pos, is, states);
       else if (token == "d")     sync_cout << pos << sync_endl;
       else if (token == "eval")  sync_cout << Eval::trace(pos) << sync_endl;
       else if (token == "perft")
@@ -202,7 +244,7 @@ void UCI::loop(int argc, char* argv[]) {
           ss << Options["Hash"] << " " << Options["Threads"]
              << " " << depth << " current perft";
 
-          benchmark(pos, ss);
+          // TODO benchmark(pos, ss);
       }
       else
           sync_cout << "Unknown command: " << cmd << sync_endl;