Skip to content

Commit

Permalink
Try a backup copy if the downloaded cutechess-cli is not working.
Browse files Browse the repository at this point in the history 
Logic:

1) Check if cutechess-cli exists and is working.

2) If not: download it.

3) Check if downloaded cutechess-cli is working.

4) If not: try to restore a backup copy.

5) If successful check if the backup copy is working.

6) If not: bail out.

Do not treat MacOS specially in the updater.

Move cutechess-cli setup to worker.py so that it is done
only once.
  • Loading branch information
@vdbergh
vdbergh committed Jun 27, 2022
1 parent 6c283d9 commit 0d30dcc
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 73 deletions.
72 changes: 3 additions & 69 deletions worker/games.py
View file
Expand Up @@ -257,49 +257,6 @@ def required_net(engine):
return net


def verify_required_cutechess(testing_dir, cutechess):
print(
"Obtaining version info for {} ...".format(os.path.join(testing_dir, cutechess))
)
os.chdir(testing_dir)
try:
with subprocess.Popen(
[os.path.join(".", cutechess), "--version"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=1,
close_fds=not IS_WINDOWS,
) as p:
errors = p.stderr.read()
pattern = re.compile("cutechess-cli ([0-9]*).([0-9]*).([0-9]*)")
major, minor, patch = 0, 0, 0
for line in iter(p.stdout.readline, ""):
m = pattern.search(line)
if m:
print("Found: ", line.strip())
major = int(m.group(1))
minor = int(m.group(2))
patch = int(m.group(3))
except (OSError, subprocess.SubprocessError) as e:
raise FatalException("Unable to run cutechess-cl. Error: {}".format(str(e)))

if p.returncode != 0:
raise FatalException(
"Unable to run cutechess-cli. Return code: {}. Error: {}".format(
format_return_code(p.returncode), errors
)
)

if major + minor + patch == 0:
raise FatalException("Unable to find the version of cutechess-cli.")

if (major, minor) < (1, 2):
raise FatalException(
"Requires cutechess 1.2 or higher, found version doesn't match"
)


def required_net_from_source():
"""Parse evaluate.h and ucioption.cpp to find default net"""
net = None
Expand Down Expand Up @@ -1146,7 +1103,7 @@ def run_games(worker_info, password, remote, run, task_id, pgn_file):
if "start" in task:
print("Variable task sizes used. Opening offset = {}".format(opening_offset))
start_game_index = opening_offset + input_total_games
run_seed = int(hashlib.sha1(run["_id"].encode("utf-8")).hexdigest(), 16) % (2**30)
run_seed = int(hashlib.sha1(run["_id"].encode("utf-8")).hexdigest(), 16) % (2 ** 30)

# Format options according to cutechess syntax.
def parse_options(s):
Expand All @@ -1164,33 +1121,9 @@ def parse_options(s):
new_options = parse_options(new_options)
base_options = parse_options(base_options)

# Create the testing directory if missing.
# Clean up old engines (keeping the num_bkps most recent).
worker_dir = os.path.dirname(os.path.realpath(__file__))
testing_dir = os.path.join(worker_dir, "testing")
if not os.path.exists(testing_dir):
os.makedirs(testing_dir)

# Download cutechess if missing in the directory.
cutechess = "cutechess-cli" + EXE_SUFFIX
os.chdir(testing_dir)
if not os.path.exists(cutechess):
if len(EXE_SUFFIX) > 0:
zipball = "cutechess-cli-win.zip"
elif IS_MACOS:
zipball = "cutechess-cli-macos-64bit.zip"
else:
zipball = "cutechess-cli-linux-{}.zip".format(platform.architecture()[0])
download_from_github(zipball, testing_dir)
zip_file = ZipFile(zipball)
zip_file.extractall()
zip_file.close()
os.remove(zipball)
os.chmod(cutechess, os.stat(cutechess).st_mode | stat.S_IEXEC)

# Verify that cutechess is working and has the required minimum version.
verify_required_cutechess(testing_dir, cutechess)

# Clean up old engines (keeping the num_bkps most recent).
engines = glob.glob(os.path.join(testing_dir, "stockfish_*" + EXE_SUFFIX))
num_bkps = 50
if len(engines) > num_bkps:
Expand Down Expand Up @@ -1384,6 +1317,7 @@ def make_player(arg):
variant = "fischerandom"

# Run cutechess binary.
cutechess = "cutechess-cli" + EXE_SUFFIX
cmd = (
[
os.path.join(testing_dir, cutechess),
Expand Down
4 changes: 0 additions & 4 deletions worker/updater.py
View file
Expand Up @@ -68,10 +68,6 @@ def update(restart=True, test=False):
bkp_testing_dir = os.path.join(worker_dir, "_testing_" + time_stamp)
shutil.move(testing_dir, bkp_testing_dir)
os.makedirs(testing_dir)
# Copy the user built MacOS cutechess-cli
# until we will have an official MacOS build to download
if "darwin" in platform.system().lower():
shutil.copy2(os.path.join(bkp_testing_dir, "cutechess-cli"), testing_dir)
# Delete old engine binaries
engines = glob.glob(os.path.join(bkp_testing_dir, "stockfish_*"))
for engine in engines:
Expand Down
134 changes: 134 additions & 0 deletions worker/worker.py
View file
Expand Up @@ -3,6 +3,7 @@
import base64
import datetime
import getpass
import glob
import hashlib
import math
import multiprocessing
Expand All @@ -11,17 +12,20 @@
import random
import re
import signal
import stat
import subprocess
import sys
import threading
import time
import traceback
import uuid
import shutil
import zlib
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from configparser import ConfigParser
from contextlib import ExitStack
from functools import partial
from zipfile import ZipFile

# Try to import an user installed package,
# fall back to the local one in case of error.
Expand All @@ -37,10 +41,12 @@
sys.path.append(packages_dir)
import expression
from games import (
EXE_SUFFIX,
FatalException,
RunException,
WorkerException,
backup_log,
download_from_github,
log,
run_games,
send_api_post_request,
Expand Down Expand Up @@ -272,6 +278,130 @@ def get_credentials(config, options, args):
return username, password


def verify_required_cutechess(testing_dir, cutechess):
cutechess = os.path.join(testing_dir, cutechess)

print("Obtaining version info for {} ...".format(cutechess))

if not os.path.exists(cutechess):
print("{} does not exist ...".format(cutechess))
return False

try:
with subprocess.Popen(
[cutechess, "--version"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=1,
close_fds=not IS_WINDOWS,
) as p:
errors = p.stderr.read()
pattern = re.compile("cutechess-cli ([0-9]*).([0-9]*).([0-9]*)")
major, minor, patch = 0, 0, 0
for line in iter(p.stdout.readline, ""):
m = pattern.search(line)
if m:
print("Found: ", line.strip())
major = int(m.group(1))
minor = int(m.group(2))
patch = int(m.group(3))
except (OSError, subprocess.SubprocessError) as e:
print("Unable to run cutechess-cli. Error: {}".format(str(e)))
return False

if p.returncode != 0:
print(
"Unable to run cutechess-cli. Return code: {}. Error: {}".format(
format_return_code(p.returncode), errors
)
)
return False

if major + minor + patch == 0:
print("Unable to find the version of cutechess-cli.")
return False

if (major, minor) < (1, 2):
print("Requires cutechess 1.2 or higher, found version doesn't match")
return False

return True


def setup_cutechess():
# Create the testing directory if missing.
worker_dir = os.path.dirname(os.path.realpath(__file__))
testing_dir = os.path.join(worker_dir, "testing")
if not os.path.exists(testing_dir):
os.makedirs(testing_dir)

try:
os.chdir(testing_dir)
except Exception as e:
print("Unable to enter {}. Error: {}".format(testing_dir, str(e)))
return False

cutechess = "cutechess-cli" + EXE_SUFFIX

if not verify_required_cutechess(testing_dir, cutechess):
if len(EXE_SUFFIX) > 0:
zipball = "cutechess-cli-win.zip"
elif IS_MACOS:
zipball = "cutechess-cli-macos-64bit.zip"
else:
zipball = "cutechess-cli-linux-{}.zip".format(platform.architecture()[0])
try:
download_from_github(zipball, testing_dir)
zip_file = ZipFile(zipball)
zip_file.extractall()
zip_file.close()
os.remove(zipball)
os.chmod(cutechess, os.stat(cutechess).st_mode | stat.S_IEXEC)
except Exception as e:
print(
"Exception downloading or extracting {}:\n".format(zipball),
e,
sep="",
file=sys.stderr,
)

# Verify that cutechess is working and has the required minimum version.
if not verify_required_cutechess(testing_dir, cutechess):
print(
"The downloaded cutechess-cli is not working. Trying to restore a backup copy ..."
)
bkp_cutechess_clis = sorted(
glob.glob(os.path.join(worker_dir, "_testing_*", cutechess)),
key=os.path.getctime,
)
if bkp_cutechess_clis:
bkp_cutechess_cli = bkp_cutechess_clis[-1]
try:
shutil.copy(bkp_cutechess_cli, testing_dir)
except Exception as e:
print(
"Unable to copy {} to {}. Error: {}".format(
bkp_cutechess_cli, testing_dir, str(e)
)
)

if not verify_required_cutechess(testing_dir, cutechess):
print(
"The backup copy {} doesn't work either ...".format(
bkp_cutechess_cli
)
)
print("No suitable cutechess-cli found")
return False
else:
print("No backup copy found")
print("No suitable cutechess-cli found")
return False

return True


def validate(config, schema):
for v in schema:
if not config.has_section(v[0]):
Expand Down Expand Up @@ -1201,6 +1331,10 @@ def worker():
if options.only_config:
return 0

# Make sure we have a working cutechess-cli
if not setup_cutechess():
return False

# Assemble the config/options data as well as some other data in a
# "worker_info" dictionary.
# This data will be sent to the server when a new task is requested.
Expand Down

0 comments on commit 0d30dcc

Please sign in to comment.