Skip to content

Commit

Permalink
Merge the 3 repeating Python binary compilations (#28500)
Browse files Browse the repository at this point in the history 
* Merge the 3 repeating Python binary compilations

* Restore grpcio_metadata.py

* run_tests.py is running on <3.6

* Restore the Windows gevent version pin
  • Loading branch information
@lidizheng
lidizheng committed Jan 12, 2022
1 parent 9ffbc2d commit 2d4f3c5
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 95 deletions.
23 changes: 11 additions & 12 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,23 @@ libs
objs

# Python items
cython_debug/
python_build/
yapf_virtual_environment/
python_pylint_venv/
.coverage*
.eggs
htmlcov/
dist/
.pytype
*.egg
py27_gevent/
py27_native/
py3[0-9]_gevent/
py3[0-9]_native/
*.egg-info
a.out
cython_debug/
dist/
htmlcov/
py3*/
python_build/
python_pylint_venv/
src/python/grpcio_*/=*
src/python/grpcio_*/build/
src/python/grpcio_*/LICENSE
src/python/grpcio_status/grpc_status/google/rpc/status.proto
.pytype
*.egg-info
yapf_virtual_environment/

# Node installation output
node_modules
Expand Down
11 changes: 4 additions & 7 deletions tools/run_tests/helper_scripts/build_python.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,16 +176,13 @@ pip_install_dir() {
cd "$PWD"
}

case "$VENV" in
*py36_gevent*)
# Install gevent
if [[ "$VENV" == "py36" ]]; then
# TODO(https://github.com/grpc/grpc/issues/15411) unpin this
pip_install gevent==1.3.b1
;;
*gevent*)
else
pip_install -U gevent
;;
esac

fi

pip_install --upgrade cython
pip_install --upgrade six protobuf
Expand Down
1 change: 0 additions & 1 deletion tools/run_tests/helper_scripts/run_python.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,3 @@ $PYTHON "$ROOT/src/python/grpcio_tests/setup.py" "$2"
mkdir -p "$ROOT/reports"
rm -rf "$ROOT/reports/python-coverage"
(mv -T "$ROOT/htmlcov" "$ROOT/reports/python-coverage") || true

140 changes: 66 additions & 74 deletions tools/run_tests/run_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,22 +178,18 @@ def _is_use_docker_child():
'venv_relative_python',
'toolchain',
'runner',
'test_name',
'iomgr_platform',
])


def _python_config_generator(name, major, minor, bits, config_vars):
name += '_' + config_vars.iomgr_platform
return PythonConfig(
name, config_vars.shell + config_vars.builder +
config_vars.builder_prefix_arguments +
[_python_pattern_function(major=major, minor=minor, bits=bits)] +
[name] + config_vars.venv_relative_python + config_vars.toolchain,
config_vars.shell + config_vars.runner + [
os.path.join(name, config_vars.venv_relative_python[0]),
config_vars.test_name
])
build = (config_vars.shell + config_vars.builder +
config_vars.builder_prefix_arguments +
[_python_pattern_function(major=major, minor=minor, bits=bits)] +
[name] + config_vars.venv_relative_python + config_vars.toolchain)
run = (config_vars.shell + config_vars.runner + [
os.path.join(name, config_vars.venv_relative_python[0]),
])
return PythonConfig(name, build, run)


def _pypy_config_generator(name, major, config_vars):
Expand Down Expand Up @@ -616,8 +612,9 @@ class PythonLanguage(object):
],
'asyncio': ['src/python/grpcio_tests/tests_aio/tests.json'],
}
_TEST_FOLDER = {
'native': 'test',

_TEST_COMMAND = {
'native': 'test_lite',
'gevent': 'test_gevent',
'asyncio': 'test_aio',
}
Expand All @@ -629,28 +626,36 @@ def configure(self, config, args):

def test_specs(self):
# load list of known test suites
tests_json = []
for tests_json_file_name in self._TEST_SPECS_FILE[
self.args.iomgr_platform]:
with open(tests_json_file_name) as tests_json_file:
tests_json.extend(json.load(tests_json_file))
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
# TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not
# designed for non-native IO manager. It has a side-effect that
# overrides threading settings in C-Core.
if args.iomgr_platform != 'native':
environment['GRPC_ENABLE_FORK_SUPPORT'] = '0'
return [
self.config.job_spec(
config.run,
timeout_seconds=8 * 60,
environ=dict(GRPC_PYTHON_TESTRUNNER_FILTER=str(suite_name),
**environment),
shortname='%s.%s.%s' %
(config.name, self._TEST_FOLDER[self.args.iomgr_platform],
suite_name),
) for suite_name in tests_json for config in self.pythons
]
jobs = []
for io_platform in self._TEST_SPECS_FILE:
test_cases = []
for tests_json_file_name in self._TEST_SPECS_FILE[io_platform]:
with open(tests_json_file_name) as tests_json_file:
test_cases.extend(json.load(tests_json_file))

environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
# TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not
# designed for non-native IO manager. It has a side-effect that
# overrides threading settings in C-Core.
if io_platform != 'native':
environment['GRPC_ENABLE_FORK_SUPPORT'] = '0'
for python_config in self.pythons:
# TODO(https://github.com/grpc/grpc/issues/23784) allow gevent
# to run on later version once issue solved.
if io_platform == 'gevent' and python_config.name != 'py36':
continue
jobs.extend([
self.config.job_spec(
python_config.run + [self._TEST_COMMAND[io_platform]],
timeout_seconds=8 * 60,
environ=dict(
GRPC_PYTHON_TESTRUNNER_FILTER=str(test_case),
**environment),
shortname='%s.%s.%s' %
(python_config.name, io_platform, test_case),
) for test_case in test_cases
])
return jobs

def pre_build_steps(self):
return []
Expand Down Expand Up @@ -688,6 +693,11 @@ def _python_manager_name(self):

def _get_pythons(self, args):
"""Get python runtimes to test with, based on current platform, architecture, compiler etc."""
if args.iomgr_platform != 'native':
raise ValueError(
'Python builds no longer differentiate IO Manager platforms, please use "native"'
)

if args.arch == 'x86':
bits = '32'
else:
Expand All @@ -712,25 +722,13 @@ def _get_pythons(self, args):
venv_relative_python = ['bin/python']
toolchain = ['unix']

# Selects the corresponding testing mode.
# See src/python/grpcio_tests/commands.py for implementation details.
if args.iomgr_platform == 'native':
test_command = 'test_lite'
elif args.iomgr_platform == 'gevent':
test_command = 'test_gevent'
elif args.iomgr_platform == 'asyncio':
test_command = 'test_aio'
else:
raise ValueError('Unsupported IO Manager platform: %s' %
args.iomgr_platform)
runner = [
os.path.abspath('tools/run_tests/helper_scripts/run_python.sh')
]

config_vars = _PythonConfigVars(shell, builder,
builder_prefix_arguments,
venv_relative_python, toolchain, runner,
test_command, args.iomgr_platform)
venv_relative_python, toolchain, runner)
python36_config = _python_config_generator(name='py36',
major='3',
minor='6',
Expand All @@ -751,41 +749,31 @@ def _get_pythons(self, args):
minor='9',
bits=bits,
config_vars=config_vars)
python310_config = _python_config_generator(name='py310',
major='3',
minor='10',
bits=bits,
config_vars=config_vars)
pypy27_config = _pypy_config_generator(name='pypy',
major='2',
config_vars=config_vars)
pypy32_config = _pypy_config_generator(name='pypy3',
major='3',
config_vars=config_vars)

if args.iomgr_platform in ('asyncio', 'gevent'):
if args.compiler not in ('default', 'python3.6', 'python3.7',
'python3.8', 'python3.9'):
raise Exception(
'Compiler %s not supported with IO Manager platform: %s' %
(args.compiler, args.iomgr_platform))

if args.compiler == 'default':
if os.name == 'nt':
if args.iomgr_platform == 'gevent':
# TODO(https://github.com/grpc/grpc/issues/23784) allow
# gevent to run on later version once issue solved.
return (python36_config,)
else:
return (python38_config,)
return (python38_config,)
elif os.uname()[0] == 'Darwin':
# NOTE(rbellevi): Testing takes significantly longer on
# MacOS, so we restrict the number of interpreter versions
# tested.
return (python38_config,)
else:
if args.iomgr_platform in ('asyncio', 'gevent'):
return (python36_config, python38_config)
elif os.uname()[0] == 'Darwin':
# NOTE(rbellevi): Testing takes significantly longer on
# MacOS, so we restrict the number of interpreter versions
# tested.
return (python38_config,)
else:
return (
python37_config,
python38_config,
)
return (
python36_config,
python38_config,
)
elif args.compiler == 'python3.6':
return (python36_config,)
elif args.compiler == 'python3.7':
Expand All @@ -794,6 +782,8 @@ def _get_pythons(self, args):
return (python38_config,)
elif args.compiler == 'python3.9':
return (python39_config,)
elif args.compiler == 'python3.10':
return (python310_config,)
elif args.compiler == 'pypy':
return (pypy27_config,)
elif args.compiler == 'pypy3':
Expand All @@ -805,6 +795,8 @@ def _get_pythons(self, args):
python36_config,
python37_config,
python38_config,
python39_config,
python310_config,
)
else:
raise Exception('Compiler %s not supported.' % args.compiler)
Expand Down
2 changes: 1 addition & 1 deletion tools/run_tests/run_tests_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
test_jobs += _generate_jobs(languages=['python'],
configs=['opt'],
platforms=['linux', 'macos', 'windows'],
iomgr_platforms=['native', 'gevent', 'asyncio'],
iomgr_platforms=['native'],
labels=['basictests', 'multilang'],
extra_args=extra_args +
['--report_multi_target'],
Expand Down

0 comments on commit 2d4f3c5

Please sign in to comment.