Export and clean up net downloading script

Fixes https://github.com/official-stockfish/Stockfish/issues/5564

This patch extracts the net downloading script in Makefile into an
external script file. Also the script is moderately rewritten for
improved readability and speed.

* Use wget preferentially over curl, as curl is known to have slight
  overhead.
* Use command instead of hash to check if command exists. Reportedly,
  hash always returns zero in some POSIX shells even when the command
  fails.
* Command existence checks (wget/curl, sha256sum) are performed only
  once at the beginning.
* Each of common patterns is encapsulated in a function
  (get_nnue_filename, validate_network).
* Print out error/warning messages to stderr.

closes https://github.com/official-stockfish/Stockfish/pull/5563

No functional change

Co-authored-by: Disservin <disservin.social@gmail.com>

.github/workflows/tests.yml

@@ -143,7 +143,7 @@ jobs:
           FROM ${{ matrix.config.base_image }}
           WORKDIR /app
           RUN apk update && apk add make g++
-          CMD ["sh", "script.sh"]
+          CMD ["sh", "src/script.sh"]
           EOF
 
       - name: Download required macOS packages
@@ -176,7 +176,7 @@ jobs:
             $COMPCXX -v
           else
             echo "$COMPCXX -v" > script.sh
-            docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
+            docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
           fi
 
       - name: Test help target
@@ -342,8 +342,8 @@ jobs:
       - name: Test riscv64 build
         if: matrix.config.run_riscv64_tests
         run: |
-          echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh
+          echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh
-          docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
+          docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
           ../tests/signature.sh $benchref
 
       # ppc64 tests
@@ -351,8 +351,8 @@ jobs:
       - name: Test ppc64 build
         if: matrix.config.run_ppc64_tests
         run: |
-          echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh
+          echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh
-          docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
+          docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
           ../tests/signature.sh $benchref
 
       # Other tests

scripts/net.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+wget_or_curl=$( (command -v wget > /dev/null 2>&1 && echo "wget -q") || \
+                (command -v curl > /dev/null 2>&1 && echo "curl -L -s -k"))
+
+if [ -z "$wget_or_curl" ]; then
+  >&2 printf "%s\n" "Neither wget or curl is installed." \
+	         "Install one of these tools to download NNUE files automatically."
+  exit 1
+fi
+
+sha256sum=$( (command -v shasum > /dev/null 2>&1 && echo "shasum -a 256") || \
+             (command -v sha256sum > /dev/null 2>&1 && echo "sha256sum"))
+
+if [ -z "$sha256sum" ]; then
+  >&2 echo "sha256sum not found, NNUE files will be assumed valid."
+fi
+
+get_nnue_filename() {
+  grep "$1" evaluate.h | grep "#define" | sed "s/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/"
+}
+
+validate_network() {
+  # If no sha256sum command is available, assume the file is always valid.
+  if [ -n "$sha256sum" ] && [ -f "$1" ]; then
+    if [ "$1" != "nn-$($sha256sum "$1" | cut -c 1-12).nnue" ]; then
+      rm -f "$1"
+      return 1
+    fi
+  fi
+}
+
+fetch_network() {
+  _filename="$(get_nnue_filename "$1")"
+
+  if [ -z "$_filename" ]; then
+    >&2 echo "NNUE file name not found for: $1"
+    return 1
+  fi
+
+  if [ -f "$_filename" ]; then
+    if validate_network "$_filename"; then
+      echo "Existing $_filename validated, skipping download"
+      return
+    else
+      echo "Removing invalid NNUE file: $_filename"
+    fi
+  fi
+
+  for url in \
+    "https://tests.stockfishchess.org/api/nn/$_filename" \
+    "https://github.com/official-stockfish/networks/raw/master/$_filename"; do
+    echo "Downloading from $url ..."
+    if $wget_or_curl "$url"; then
+      if validate_network "$_filename"; then
+        echo "Successfully validated $_filename"
+      else
+        echo "Downloaded $_filename is invalid"
+        continue
+      fi
+    else
+      echo "Failed to download from $url"
+    fi
+    if [ -f "$_filename" ]; then
+      return
+    fi
+  done
+
+  # Download was not successful in the loop, return false.
+  >&2 echo "Failed to download $_filename"
+  return 1
+}
+
+fetch_network EvalFileDefaultNameBig && \
+fetch_network EvalFileDefaultNameSmall

src/Makefile

@@ -917,59 +917,9 @@ profileclean:
 	@rm -f stockfish.res
 	@rm -f ./-lstdc++.res
 
-define fetch_network
-	@echo "Default net: $(nnuenet)"
-	@if [ "x$(curl_or_wget)" = "x" ]; then \
-		echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \
-	fi
-	@if [ "x$(shasum_command)" = "x" ]; then \
-		echo "shasum / sha256sum not found, skipping net validation"; \
-	elif test -f "$(nnuenet)"; then \
-		if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
-			echo "Removing invalid network"; rm -f $(nnuenet); \
-		fi; \
-	fi;
-	@for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \
-		if test -f "$(nnuenet)"; then \
-			echo "$(nnuenet) available : OK"; break; \
-		else \
-			if [ "x$(curl_or_wget)" != "x" ]; then \
-				echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\
-			else \
-				echo "No net found and download not possible"; exit 1;\
-			fi; \
-		fi; \
-		if [ "x$(shasum_command)" != "x" ]; then \
-			if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
-				echo "Removing failed download"; rm -f $(nnuenet); \
-			fi; \
-		fi; \
-	done
-	@if ! test -f "$(nnuenet)"; then \
-		echo "Failed to download $(nnuenet)."; \
-	fi;
-	@if [ "x$(shasum_command)" != "x" ]; then \
-		if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
-			echo "Network validated"; break; \
-		fi; \
-	fi;
-endef
-
-# set up shell variables for the net stuff
-define netvariables
-$(eval nnuenet := $(shell grep $(1) evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
-$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
-$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
-$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
-$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
-endef
-
 # evaluation network (nnue)
 net:
-	$(call netvariables, EvalFileDefaultNameBig)
+	@$(SHELL) ../scripts/net.sh
-	$(call fetch_network)
-	$(call netvariables, EvalFileDefaultNameSmall)
-	$(call fetch_network)
 
 format:
 	$(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file