mirror of
https://github.com/sockspls/badfish
synced 2025-04-29 08:13:08 +00:00
Export and clean up net downloading script
Fixes https://github.com/official-stockfish/Stockfish/issues/5564 This patch extracts the net downloading script in Makefile into an external script file. Also the script is moderately rewritten for improved readability and speed. * Use wget preferentially over curl, as curl is known to have slight overhead. * Use command instead of hash to check if command exists. Reportedly, hash always returns zero in some POSIX shells even when the command fails. * Command existence checks (wget/curl, sha256sum) are performed only once at the beginning. * Each of common patterns is encapsulated in a function (get_nnue_filename, validate_network). * Print out error/warning messages to stderr. closes https://github.com/official-stockfish/Stockfish/pull/5563 No functional change Co-authored-by: Disservin <disservin.social@gmail.com>
This commit is contained in:
MinetaS
Disservin
parent
66a7965b0f
commit
1b310cc87e
3 changed files with 82 additions and 57 deletions
12
.github/workflows/tests.yml
vendored
12
.github/workflows/tests.yml
vendored
|
|
@ -143,7 +143,7 @@ jobs:
|
|||
FROM ${{ matrix.config.base_image }}
|
||||
WORKDIR /app
|
||||
RUN apk update && apk add make g++
|
||||
CMD ["sh", "script.sh"]
|
||||
CMD ["sh", "src/script.sh"]
|
||||
EOF
|
||||
|
||||
- name: Download required macOS packages
|
||||
|
|
@ -176,7 +176,7 @@ jobs:
|
|||
$COMPCXX -v
|
||||
else
|
||||
echo "$COMPCXX -v" > script.sh
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
|
||||
fi
|
||||
|
||||
- name: Test help target
|
||||
|
|
@ -342,8 +342,8 @@ jobs:
|
|||
- name: Test riscv64 build
|
||||
if: matrix.config.run_riscv64_tests
|
||||
run: |
|
||||
echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
|
||||
echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=riscv64 build" > script.sh
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
|
||||
../tests/signature.sh $benchref
|
||||
|
||||
# ppc64 tests
|
||||
|
|
@ -351,8 +351,8 @@ jobs:
|
|||
- name: Test ppc64 build
|
||||
if: matrix.config.run_ppc64_tests
|
||||
run: |
|
||||
echo "export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}/src:/app sf_builder
|
||||
echo "cd src && export LDFLAGS='-static' && make clean && make -j4 ARCH=ppc-64 build" > script.sh
|
||||
docker run --rm --platform ${{ matrix.config.platform }} -v ${{ github.workspace }}:/app sf_builder
|
||||
../tests/signature.sh $benchref
|
||||
|
||||
# Other tests
|
||||
|
|
|
|||
75
scripts/net.sh
Executable file
75
scripts/net.sh
Executable file
|
|
@ -0,0 +1,75 @@
|
|||
#!/bin/sh
|
||||
|
||||
wget_or_curl=$( (command -v wget > /dev/null 2>&1 && echo "wget -q") || \
|
||||
(command -v curl > /dev/null 2>&1 && echo "curl -L -s -k"))
|
||||
|
||||
if [ -z "$wget_or_curl" ]; then
|
||||
>&2 printf "%s\n" "Neither wget or curl is installed." \
|
||||
"Install one of these tools to download NNUE files automatically."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sha256sum=$( (command -v shasum > /dev/null 2>&1 && echo "shasum -a 256") || \
|
||||
(command -v sha256sum > /dev/null 2>&1 && echo "sha256sum"))
|
||||
|
||||
if [ -z "$sha256sum" ]; then
|
||||
>&2 echo "sha256sum not found, NNUE files will be assumed valid."
|
||||
fi
|
||||
|
||||
get_nnue_filename() {
|
||||
grep "$1" evaluate.h | grep "#define" | sed "s/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/"
|
||||
}
|
||||
|
||||
validate_network() {
|
||||
# If no sha256sum command is available, assume the file is always valid.
|
||||
if [ -n "$sha256sum" ] && [ -f "$1" ]; then
|
||||
if [ "$1" != "nn-$($sha256sum "$1" | cut -c 1-12).nnue" ]; then
|
||||
rm -f "$1"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_network() {
|
||||
_filename="$(get_nnue_filename "$1")"
|
||||
|
||||
if [ -z "$_filename" ]; then
|
||||
>&2 echo "NNUE file name not found for: $1"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -f "$_filename" ]; then
|
||||
if validate_network "$_filename"; then
|
||||
echo "Existing $_filename validated, skipping download"
|
||||
return
|
||||
else
|
||||
echo "Removing invalid NNUE file: $_filename"
|
||||
fi
|
||||
fi
|
||||
|
||||
for url in \
|
||||
"https://tests.stockfishchess.org/api/nn/$_filename" \
|
||||
"https://github.com/official-stockfish/networks/raw/master/$_filename"; do
|
||||
echo "Downloading from $url ..."
|
||||
if $wget_or_curl "$url"; then
|
||||
if validate_network "$_filename"; then
|
||||
echo "Successfully validated $_filename"
|
||||
else
|
||||
echo "Downloaded $_filename is invalid"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
echo "Failed to download from $url"
|
||||
fi
|
||||
if [ -f "$_filename" ]; then
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
# Download was not successful in the loop, return false.
|
||||
>&2 echo "Failed to download $_filename"
|
||||
return 1
|
||||
}
|
||||
|
||||
fetch_network EvalFileDefaultNameBig && \
|
||||
fetch_network EvalFileDefaultNameSmall
|
||||
52
src/Makefile
52
src/Makefile
|
|
@ -917,59 +917,9 @@ profileclean:
|
|||
@rm -f stockfish.res
|
||||
@rm -f ./-lstdc++.res
|
||||
|
||||
define fetch_network
|
||||
@echo "Default net: $(nnuenet)"
|
||||
@if [ "x$(curl_or_wget)" = "x" ]; then \
|
||||
echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \
|
||||
fi
|
||||
@if [ "x$(shasum_command)" = "x" ]; then \
|
||||
echo "shasum / sha256sum not found, skipping net validation"; \
|
||||
elif test -f "$(nnuenet)"; then \
|
||||
if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
|
||||
echo "Removing invalid network"; rm -f $(nnuenet); \
|
||||
fi; \
|
||||
fi;
|
||||
@for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \
|
||||
if test -f "$(nnuenet)"; then \
|
||||
echo "$(nnuenet) available : OK"; break; \
|
||||
else \
|
||||
if [ "x$(curl_or_wget)" != "x" ]; then \
|
||||
echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\
|
||||
else \
|
||||
echo "No net found and download not possible"; exit 1;\
|
||||
fi; \
|
||||
fi; \
|
||||
if [ "x$(shasum_command)" != "x" ]; then \
|
||||
if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
|
||||
echo "Removing failed download"; rm -f $(nnuenet); \
|
||||
fi; \
|
||||
fi; \
|
||||
done
|
||||
@if ! test -f "$(nnuenet)"; then \
|
||||
echo "Failed to download $(nnuenet)."; \
|
||||
fi;
|
||||
@if [ "x$(shasum_command)" != "x" ]; then \
|
||||
if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
|
||||
echo "Network validated"; break; \
|
||||
fi; \
|
||||
fi;
|
||||
endef
|
||||
|
||||
# set up shell variables for the net stuff
|
||||
define netvariables
|
||||
$(eval nnuenet := $(shell grep $(1) evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
|
||||
$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
|
||||
$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
|
||||
$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
|
||||
$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
|
||||
endef
|
||||
|
||||
# evaluation network (nnue)
|
||||
net:
|
||||
$(call netvariables, EvalFileDefaultNameBig)
|
||||
$(call fetch_network)
|
||||
$(call netvariables, EvalFileDefaultNameSmall)
|
||||
$(call fetch_network)
|
||||
@$(SHELL) ../scripts/net.sh
|
||||
|
||||
format:
|
||||
$(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue