From 1cbdd26a0bdd1f8231449e1f389f3c804529bc35 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 4 Oct 2020 19:38:25 +0300 Subject: [PATCH] The celery multi command now works as expected. --- celery/apps/multi.py | 26 +++++++++++++++++++++--- celery/bin/multi.py | 7 ++++++- requirements/test-ci-default.txt | 2 +- t/unit/apps/test_multi.py | 35 ++++++++++++++++---------------- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/celery/apps/multi.py b/celery/apps/multi.py index b82eee4c9b..a1458e3bd6 100644 --- a/celery/apps/multi.py +++ b/celery/apps/multi.py @@ -150,7 +150,7 @@ def _setdefaultopt(self, d, alt, value): pass value = d.setdefault(alt[0], os.path.normpath(value)) dir_path = os.path.dirname(value) - if not os.path.exists(dir_path): + if dir_path and not os.path.exists(dir_path): os.makedirs(dir_path) return value @@ -160,10 +160,30 @@ def _prepare_expander(self): self.name, shortname, hostname) def _prepare_argv(self): + cmd = self.expander(self.cmd).split(' ') + i = cmd.index('celery') + 1 + + options = self.options.copy() + for opt, value in self.options.items(): + if opt in ( + '-A', '--app', + '-b', '--broker', + '--result-backend', + '--loader', + '--config', + '--workdir', + '-C', '--no-color', + '-q', '--quiet', + ): + cmd.insert(i, format_opt(opt, self.expander(value))) + + options.pop(opt) + + cmd = [' '.join(cmd)] argv = tuple( - [self.expander(self.cmd)] + + cmd + [format_opt(opt, self.expander(value)) - for opt, value in self.options.items()] + + for opt, value in options.items()] + [self.extra_args] ) if self.append: diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 3e999ab2ab..12bb52b87d 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -471,4 +471,9 @@ def DOWN(self): def multi(ctx): """Start multiple worker instances.""" cmd = MultiTool(quiet=ctx.obj.quiet, no_color=ctx.obj.no_color) - return cmd.execute_from_commandline([''] + ctx.args) + # In 4.x, celery multi ignores the global --app option. + # Since in 5.0 the --app option is global only we + # rearrange the arguments so that the MultiTool will parse them correctly. + args = sys.argv[1:] + args = args[args.index('multi'):] + args[:args.index('multi')] + return cmd.execute_from_commandline(args) diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index 953ed9aecc..fdcf468473 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -12,7 +12,7 @@ -r extras/thread.txt -r extras/elasticsearch.txt -r extras/couchdb.txt --r extras/couchbase.txt +#-r extras/couchbase.txt -r extras/arangodb.txt -r extras/consul.txt -r extras/cosmosdbsql.txt diff --git a/t/unit/apps/test_multi.py b/t/unit/apps/test_multi.py index f7de1d5e27..4c3fd9bfc1 100644 --- a/t/unit/apps/test_multi.py +++ b/t/unit/apps/test_multi.py @@ -69,7 +69,7 @@ def test_parse(self, gethostname, mkdirs_mock): '--', '.disable_rate_limits=1', ]) p.parse() - it = multi_args(p, cmd='COMMAND', append='*AP*', + it = multi_args(p, cmd='celery multi', append='*AP*', prefix='*P*', suffix='*S*') nodes = list(it) @@ -85,32 +85,32 @@ def assert_line_in(name, args): assert_line_in( '*P*jerry@*S*', - ['COMMAND', '-n *P*jerry@*S*', '-Q bar', + ['celery multi', '-n *P*jerry@*S*', '-Q bar', '-c 5', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) assert_line_in( '*P*elaine@*S*', - ['COMMAND', '-n *P*elaine@*S*', '-Q bar', + ['celery multi', '-n *P*elaine@*S*', '-Q bar', '-c 5', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) assert_line_in( '*P*kramer@*S*', - ['COMMAND', '--loglevel=DEBUG', '-n *P*kramer@*S*', + ['celery multi', '--loglevel=DEBUG', '-n *P*kramer@*S*', '-Q bar', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) expand = nodes[0].expander assert expand('%h') == '*P*jerry@*S*' assert expand('%n') == '*P*jerry' - nodes2 = list(multi_args(p, cmd='COMMAND', append='', + nodes2 = list(multi_args(p, cmd='celery multi', append='', prefix='*P*', suffix='*S*')) assert nodes2[0].argv[-1] == '-- .disable_rate_limits=1' p2 = NamespacedOptionParser(['10', '-c:1', '5']) p2.parse() - nodes3 = list(multi_args(p2, cmd='COMMAND')) + nodes3 = list(multi_args(p2, cmd='celery multi')) def _args(name, *args): return args + ( @@ -123,40 +123,40 @@ def _args(name, *args): assert len(nodes3) == 10 assert nodes3[0].name == 'celery1@example.com' assert nodes3[0].argv == ( - 'COMMAND', '-c 5', '-n celery1@example.com') + _args('celery1') + 'celery multi', '-c 5', '-n celery1@example.com') + _args('celery1') for i, worker in enumerate(nodes3[1:]): assert worker.name == 'celery%s@example.com' % (i + 2) node_i = f'celery{i + 2}' assert worker.argv == ( - 'COMMAND', + 'celery multi', f'-n {node_i}@example.com') + _args(node_i) - nodes4 = list(multi_args(p2, cmd='COMMAND', suffix='""')) + nodes4 = list(multi_args(p2, cmd='celery multi', suffix='""')) assert len(nodes4) == 10 assert nodes4[0].name == 'celery1@' assert nodes4[0].argv == ( - 'COMMAND', '-c 5', '-n celery1@') + _args('celery1') + 'celery multi', '-c 5', '-n celery1@') + _args('celery1') p3 = NamespacedOptionParser(['foo@', '-c:foo', '5']) p3.parse() - nodes5 = list(multi_args(p3, cmd='COMMAND', suffix='""')) + nodes5 = list(multi_args(p3, cmd='celery multi', suffix='""')) assert nodes5[0].name == 'foo@' assert nodes5[0].argv == ( - 'COMMAND', '-c 5', '-n foo@') + _args('foo') + 'celery multi', '-c 5', '-n foo@') + _args('foo') p4 = NamespacedOptionParser(['foo', '-Q:1', 'test']) p4.parse() - nodes6 = list(multi_args(p4, cmd='COMMAND', suffix='""')) + nodes6 = list(multi_args(p4, cmd='celery multi', suffix='""')) assert nodes6[0].name == 'foo@' assert nodes6[0].argv == ( - 'COMMAND', '-Q test', '-n foo@') + _args('foo') + 'celery multi', '-Q test', '-n foo@') + _args('foo') p5 = NamespacedOptionParser(['foo@bar', '-Q:1', 'test']) p5.parse() - nodes7 = list(multi_args(p5, cmd='COMMAND', suffix='""')) + nodes7 = list(multi_args(p5, cmd='celery multi', suffix='""')) assert nodes7[0].name == 'foo@bar' assert nodes7[0].argv == ( - 'COMMAND', '-Q test', '-n foo@bar') + _args('foo') + 'celery multi', '-Q test', '-n foo@bar') + _args('foo') p6 = NamespacedOptionParser(['foo@bar', '-Q:0', 'test']) p6.parse() @@ -192,8 +192,7 @@ def test_from_kwargs(self): max_tasks_per_child=30, A='foo', Q='q1,q2', O='fair', ) assert sorted(n.argv) == sorted([ - '-m celery worker --detach', - '-A foo', + '-m celery -A foo worker --detach', f'--executable={n.executable}', '-O fair', '-n foo@bar.com',