From ba1c63983694353cec7e395ccbc4282cee7efa14 Mon Sep 17 00:00:00 2001
From: Joost VandeVondele <Joost.VandeVondele@gmail.com>
Date: Sat, 15 Dec 2018 09:00:19 +0100
Subject: [PATCH] [cluster] fill sendbuffer better

use a counter to track available elements.

Some elo gain, on 4 ranks:

Score of old-r4-1t vs new-r4-1t: 422 - 508 - 1694  [0.484] 2624
Elo difference: -11.39 +/- 7.90
---
 src/cluster.cpp | 8 +++++---
 src/cluster.h   | 2 +-
 src/thread.h    | 1 +
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/cluster.cpp b/src/cluster.cpp
index 681cb4ed..8df03b7c 100644
--- a/src/cluster.cpp
+++ b/src/cluster.cpp
@@ -176,16 +176,15 @@ void save(Thread* thread, TTEntry* tte,
      {
          std::lock_guard<Mutex> lk(thread->ttBuffer.mutex);
          thread->ttBuffer.buffer.replace(KeyedTTEntry(k,*tte));
+         ++thread->ttBuffer.counter;
      }
 
      // Communicate on main search thread
-     if (thread == Threads.main())
+     if (thread == Threads.main() && thread->ttBuffer.counter * Threads.size() > TTSendBufferSize)
      {
          static MPI_Request req = MPI_REQUEST_NULL;
          static TTSendBuffer<TTSendBufferSize> send_buff = {};
          int flag;
-         bool found;
-         TTEntry* replace_tte;
 
          // Test communication status
          MPI_Test(&req, &flag, MPI_STATUS_IGNORE);
@@ -202,6 +201,8 @@ void save(Thread* thread, TTEntry* tte,
                  for (size_t i = irank * TTSendBufferSize ; i < (irank + 1) * TTSendBufferSize; ++i)
                  {
                      auto&& e = TTBuff[i];
+                     bool found;
+                     TTEntry* replace_tte;
                      replace_tte = TT.probe(e.first, found);
                      replace_tte->save(e.first, e.second.value(), e.second.bound(), e.second.depth(),
                                        e.second.move(), e.second.eval());
@@ -219,6 +220,7 @@ void save(Thread* thread, TTEntry* tte,
                      send_buff.replace(e);
                  // Reset thread's send buffer
                  th->ttBuffer.buffer = {};
+                 th->ttBuffer.counter = 0;
              }
 
              // Start next communication
diff --git a/src/cluster.h b/src/cluster.h
index 538a5479..854b809d 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -42,7 +42,7 @@ struct MoveInfo {
 #ifdef USE_MPI
 using KeyedTTEntry = std::pair<Key, TTEntry>;
 
-constexpr std::size_t TTSendBufferSize = 16;
+constexpr std::size_t TTSendBufferSize = 32;
 template <std::size_t N> class TTSendBuffer : public std::array<KeyedTTEntry, N> {
 
   struct Compare {
diff --git a/src/thread.h b/src/thread.h
index 200f0df8..4f34de51 100644
--- a/src/thread.h
+++ b/src/thread.h
@@ -79,6 +79,7 @@ public:
   struct {
       Mutex mutex;
       Cluster::TTSendBuffer<Cluster::TTSendBufferSize> buffer = {};
+      size_t counter = 0;
   } ttBuffer;
 #endif
 };