From b01fca448bd0685561badc5f19b178f091ed4590 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 30 Jul 2021 12:27:08 -0400 Subject: [PATCH 01/14] Support custom stdout callbacks --- ansible_runner/callbacks/awx_display.py | 6 +-- ansible_runner/callbacks/awx_minimal.py | 18 ++++++++ ansible_runner/callbacks/minimal.py | 50 ---------------------- ansible_runner/config/_base.py | 4 -- ansible_runner/config/runner.py | 6 +++ ansible_runner/display_callback/minimal.py | 29 ------------- ansible_runner/display_callback/module.py | 17 ++++++-- test/unit/config/test__base.py | 1 - 8 files changed, 41 insertions(+), 90 deletions(-) create mode 100644 ansible_runner/callbacks/awx_minimal.py delete mode 100644 ansible_runner/callbacks/minimal.py delete mode 100644 ansible_runner/display_callback/minimal.py diff --git a/ansible_runner/callbacks/awx_display.py b/ansible_runner/callbacks/awx_display.py index cf877d95a..157ddc474 100644 --- a/ansible_runner/callbacks/awx_display.py +++ b/ansible_runner/callbacks/awx_display.py @@ -36,9 +36,9 @@ import sys # noqa # Add awx/lib to sys.path. -awx_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -if awx_lib_path not in sys.path: - sys.path.insert(0, awx_lib_path) +callback_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if callback_lib_path not in sys.path: + sys.path.insert(0, callback_lib_path) # Tower Display Callback from display_callback import AWXDefaultCallbackModule # noqa diff --git a/ansible_runner/callbacks/awx_minimal.py b/ansible_runner/callbacks/awx_minimal.py new file mode 100644 index 000000000..4237a1a1c --- /dev/null +++ b/ansible_runner/callbacks/awx_minimal.py @@ -0,0 +1,18 @@ +from __future__ import (absolute_import, division, print_function) + +# Python +import os # noqa +import sys # noqa + +callback_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if callback_lib_path not in sys.path: + sys.path.insert(0, callback_lib_path) + + +from display_callback import AWXMinimalCallbackModule # noqa + + +# In order to be recognized correctly, self.__class__.__name__ needs to +# match "CallbackModule" +class CallbackModule(AWXMinimalCallbackModule): + pass diff --git a/ansible_runner/callbacks/minimal.py b/ansible_runner/callbacks/minimal.py deleted file mode 100644 index b170f08ce..000000000 --- a/ansible_runner/callbacks/minimal.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - - -DOCUMENTATION = ''' - callback: minimal - short_description: Ad hoc event dispatcher for ansible-runner - version_added: "2.0" - description: - - This callback is necessary for ansible-runner to work - type: stdout - extends_documentation_fragment: - - default_callback - requirements: - - Set as stdout in config -''' - -# Python -import os # noqa -import sys # noqa - -# Add awx/lib to sys.path. -awx_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -if awx_lib_path not in sys.path: - sys.path.insert(0, awx_lib_path) - -# Tower Display Callback -from display_callback import AWXMinimalCallbackModule # noqa - - -# In order to be recognized correctly, self.__class__.__name__ needs to -# match "CallbackModule" -class CallbackModule(AWXMinimalCallbackModule): - pass diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 38ee9583a..30e1e1d38 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -265,10 +265,6 @@ def _prepare_env(self, runner_mode='pexpect'): python_path += ':' self.env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(filter(None, (self.env.get('ANSIBLE_CALLBACK_PLUGINS'), callback_dir))) - if 'AD_HOC_COMMAND_ID' in self.env: - self.env['ANSIBLE_STDOUT_CALLBACK'] = 'minimal' - else: - self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' self.env['ANSIBLE_RETRY_FILES_ENABLED'] = 'False' if 'ANSIBLE_HOST_KEY_CHECKING' not in self.env: self.env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' diff --git a/ansible_runner/config/runner.py b/ansible_runner/config/runner.py index c542e0b2d..e675688af 100644 --- a/ansible_runner/config/runner.py +++ b/ansible_runner/config/runner.py @@ -147,6 +147,11 @@ def prepare(self): self.prepare_inventory() self.prepare_command() + if self.execution_mode == ExecutionMode.ANSIBLE: + self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_minimal' + else: + self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' + if self.execution_mode == ExecutionMode.ANSIBLE_PLAYBOOK and self.playbook is None: raise ConfigurationError("Runner playbook required when running ansible-playbook") elif self.execution_mode == ExecutionMode.ANSIBLE and self.module is None: @@ -242,6 +247,7 @@ def prepare_env(self): self.env["RUNNER_OMIT_EVENTS"] = str(self.omit_event_data) self.env["RUNNER_ONLY_FAILED_EVENTS"] = str(self.only_failed_event_data) + self.env["ANSIBLE_LOAD_CALLBACK_PLUGINS"] = '1' def prepare_command(self): try: diff --git a/ansible_runner/display_callback/minimal.py b/ansible_runner/display_callback/minimal.py deleted file mode 100644 index 98076ba27..000000000 --- a/ansible_runner/display_callback/minimal.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2016 Ansible by Red Hat, Inc. -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - -# Python -import os - -# Ansible -import ansible - -# Because of the way Ansible loads plugins, it's not possible to import -# ansible.plugins.callback.minimal when being loaded as the minimal plugin. Ugh. -minimal_plugin = os.path.join(os.path.dirname(ansible.__file__), 'plugins', 'callback', 'minimal.py') -exec(compile(open(minimal_plugin, "rb").read(), minimal_plugin, 'exec')) diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/module.py index 1d9309f84..0057530f7 100644 --- a/ansible_runner/display_callback/module.py +++ b/ansible_runner/display_callback/module.py @@ -21,6 +21,7 @@ import collections import contextlib import datetime +import os import sys import uuid from copy import copy @@ -28,11 +29,21 @@ # Ansible from ansible import constants as C from ansible.plugins.callback import CallbackBase -from ansible.plugins.callback.default import CallbackModule as DefaultCallbackModule +from ansible.plugins.loader import callback_loader # AWX Display Callback from .events import event_context -from .minimal import CallbackModule as MinimalCallbackModule +MinimalCallbackModule = callback_loader.get('minimal').__class__ + +# Dynamically construct base classes for our callback module, to support +# custom stdout callbacks. +original_stdout_callback = os.environ['ANSIBLE_STDOUT_CALLBACK'] +del os.environ['ANSIBLE_STDOUT_CALLBACK'] + +default_stdout_callback = C.config.get_config_value('DEFAULT_STDOUT_CALLBACK') +DefaultCallbackModule = callback_loader.get(default_stdout_callback).__class__ + +os.environ['ANSIBLE_STDOUT_CALLBACK'] = original_stdout_callback CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa @@ -539,7 +550,7 @@ class AWXDefaultCallbackModule(BaseCallbackModule, DefaultCallbackModule): class AWXMinimalCallbackModule(BaseCallbackModule, MinimalCallbackModule): - CALLBACK_NAME = 'minimal' + CALLBACK_NAME = 'awx_minimal' def v2_playbook_on_play_start(self, play): pass diff --git a/test/unit/config/test__base.py b/test/unit/config/test__base.py index 04f3d86ee..3c1c1eb37 100644 --- a/test/unit/config/test__base.py +++ b/test/unit/config/test__base.py @@ -191,7 +191,6 @@ def test_prepare_env_ansible_vars(mocker, tmp_path): assert not hasattr(rc, 'ssh_key_path') assert not hasattr(rc, 'command') - assert rc.env['ANSIBLE_STDOUT_CALLBACK'] == 'awx_display' assert rc.env['ANSIBLE_RETRY_FILES_ENABLED'] == 'False' assert rc.env['ANSIBLE_HOST_KEY_CHECKING'] == 'False' assert rc.env['AWX_ISOLATED_DATA_DIR'] == artifact_dir.joinpath(rc.ident).as_posix() From 6b83c6b9e6d10a3485654ff440767969ef146149 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 31 Jul 2021 19:38:49 -0400 Subject: [PATCH 02/14] Refactor things down to a single stdout callback module --- ansible_runner/callbacks/awx_display.py | 4 +- ansible_runner/callbacks/awx_minimal.py | 18 --------- ansible_runner/config/runner.py | 11 ++--- ansible_runner/display_callback/__init__.py | 4 +- ansible_runner/display_callback/module.py | 40 +++++++++++-------- docs/ansible_runner.callbacks.rst | 7 ---- docs/ansible_runner.display_callback.rst | 7 ---- .../test_transmit_worker_process.py | 2 + 8 files changed, 36 insertions(+), 57 deletions(-) delete mode 100644 ansible_runner/callbacks/awx_minimal.py diff --git a/ansible_runner/callbacks/awx_display.py b/ansible_runner/callbacks/awx_display.py index 157ddc474..b66d18ca8 100644 --- a/ansible_runner/callbacks/awx_display.py +++ b/ansible_runner/callbacks/awx_display.py @@ -41,10 +41,10 @@ sys.path.insert(0, callback_lib_path) # Tower Display Callback -from display_callback import AWXDefaultCallbackModule # noqa +from display_callback import AWXCallbackModule # noqa # In order to be recognized correctly, self.__class__.__name__ needs to # match "CallbackModule" -class CallbackModule(AWXDefaultCallbackModule): +class CallbackModule(AWXCallbackModule): pass diff --git a/ansible_runner/callbacks/awx_minimal.py b/ansible_runner/callbacks/awx_minimal.py deleted file mode 100644 index 4237a1a1c..000000000 --- a/ansible_runner/callbacks/awx_minimal.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import (absolute_import, division, print_function) - -# Python -import os # noqa -import sys # noqa - -callback_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -if callback_lib_path not in sys.path: - sys.path.insert(0, callback_lib_path) - - -from display_callback import AWXMinimalCallbackModule # noqa - - -# In order to be recognized correctly, self.__class__.__name__ needs to -# match "CallbackModule" -class CallbackModule(AWXMinimalCallbackModule): - pass diff --git a/ansible_runner/config/runner.py b/ansible_runner/config/runner.py index e675688af..e2d4a8ee6 100644 --- a/ansible_runner/config/runner.py +++ b/ansible_runner/config/runner.py @@ -147,11 +147,6 @@ def prepare(self): self.prepare_inventory() self.prepare_command() - if self.execution_mode == ExecutionMode.ANSIBLE: - self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_minimal' - else: - self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' - if self.execution_mode == ExecutionMode.ANSIBLE_PLAYBOOK and self.playbook is None: raise ConfigurationError("Runner playbook required when running ansible-playbook") elif self.execution_mode == ExecutionMode.ANSIBLE and self.module is None: @@ -247,6 +242,12 @@ def prepare_env(self): self.env["RUNNER_OMIT_EVENTS"] = str(self.omit_event_data) self.env["RUNNER_ONLY_FAILED_EVENTS"] = str(self.only_failed_event_data) + + original_stdout_callback = os.getenv('ANSIBLE_STDOUT_CALLBACK') + if original_stdout_callback: + self.env['ORIGINAL_STDOUT_CALLBACK'] = original_stdout_callback + + self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' self.env["ANSIBLE_LOAD_CALLBACK_PLUGINS"] = '1' def prepare_command(self): diff --git a/ansible_runner/display_callback/__init__.py b/ansible_runner/display_callback/__init__.py index a02a2ac15..5f7e02469 100644 --- a/ansible_runner/display_callback/__init__.py +++ b/ansible_runner/display_callback/__init__.py @@ -19,6 +19,6 @@ # AWX Display Callback from . import display # noqa (wraps ansible.display.Display methods) -from .module import AWXDefaultCallbackModule, AWXMinimalCallbackModule +from .module import AWXCallbackModule -__all__ = ['AWXDefaultCallbackModule', 'AWXMinimalCallbackModule'] +__all__ = ['AWXCallbackModule'] diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/module.py index 0057530f7..7efd96ca9 100644 --- a/ansible_runner/display_callback/module.py +++ b/ansible_runner/display_callback/module.py @@ -31,19 +31,25 @@ from ansible.plugins.callback import CallbackBase from ansible.plugins.loader import callback_loader -# AWX Display Callback from .events import event_context -MinimalCallbackModule = callback_loader.get('minimal').__class__ -# Dynamically construct base classes for our callback module, to support -# custom stdout callbacks. -original_stdout_callback = os.environ['ANSIBLE_STDOUT_CALLBACK'] -del os.environ['ANSIBLE_STDOUT_CALLBACK'] +user_specified_callback = os.getenv('ORIGINAL_STDOUT_CALLBACK') +runner_callback = os.environ.pop('ANSIBLE_STDOUT_CALLBACK', 'awx_display') + +if user_specified_callback: + os.environ['ANSIBLE_STDOUT_CALLBACK'] = user_specified_callback + +is_adhoc = os.getenv('AD_HOC_COMMAND_ID', False) + +# Dynamically construct base classes for our callback module, to support custom stdout callbacks. +if is_adhoc: + default_stdout_callback = 'minimal' +else: + default_stdout_callback = C.config.get_config_value('DEFAULT_STDOUT_CALLBACK') -default_stdout_callback = C.config.get_config_value('DEFAULT_STDOUT_CALLBACK') DefaultCallbackModule = callback_loader.get(default_stdout_callback).__class__ -os.environ['ANSIBLE_STDOUT_CALLBACK'] = original_stdout_callback +os.environ['ANSIBLE_STDOUT_CALLBACK'] = runner_callback CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa @@ -543,17 +549,19 @@ def v2_runner_on_start(self, host, task): super(BaseCallbackModule, self).v2_runner_on_start(host, task) -class AWXDefaultCallbackModule(BaseCallbackModule, DefaultCallbackModule): +class AWXCallbackModule(BaseCallbackModule, DefaultCallbackModule): CALLBACK_NAME = 'awx_display' - -class AWXMinimalCallbackModule(BaseCallbackModule, MinimalCallbackModule): - - CALLBACK_NAME = 'awx_minimal' - def v2_playbook_on_play_start(self, play): - pass + if is_adhoc: + pass + else: + super().v2_playbook_on_play_start(play) + def v2_playbook_on_task_start(self, task, is_conditional): - self.set_task(task) + if is_adhoc: + self.set_task(task) + else: + super().v2_playbook_on_task_start(task, is_conditional) diff --git a/docs/ansible_runner.callbacks.rst b/docs/ansible_runner.callbacks.rst index 4a50bd48b..b2dd1787f 100644 --- a/docs/ansible_runner.callbacks.rst +++ b/docs/ansible_runner.callbacks.rst @@ -10,10 +10,3 @@ ansible_runner.callbacks.awx_display module .. automodule:: ansible_runner.callbacks.awx_display :members: :undoc-members: - -ansible_runner.callbacks.minimal module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: ansible_runner.callbacks.minimal - :members: - :undoc-members: diff --git a/docs/ansible_runner.display_callback.rst b/docs/ansible_runner.display_callback.rst index 4ad24c802..fd21c4dca 100644 --- a/docs/ansible_runner.display_callback.rst +++ b/docs/ansible_runner.display_callback.rst @@ -20,13 +20,6 @@ ansible_runner.display_callback.events module :undoc-members: :show-inheritance: -ansible_runner.display_callback.minimal module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: ansible_runner.display_callback.minimal - :members: - :undoc-members: - ansible_runner.display_callback.module module ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/test/integration/test_transmit_worker_process.py b/test/integration/test_transmit_worker_process.py index cabc3c08e..fff45441d 100644 --- a/test/integration/test_transmit_worker_process.py +++ b/test/integration/test_transmit_worker_process.py @@ -30,6 +30,8 @@ def get_job_kwargs(self, job_type): job_kwargs = dict(module='setup', host_pattern='localhost') # also test use of user env vars job_kwargs['envvars'] = dict(MY_ENV_VAR='bogus') + if job_type == 'adhoc': + job_kwargs['envvars']['AD_HOC_COMMAND_ID'] = '1' return job_kwargs def check_artifacts(self, process_dir, job_type): From 0a705c298cdf4e551e32eb6f11fd815a8baab23a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 7 Jan 2022 15:22:07 -0500 Subject: [PATCH 03/14] Condense callback to 1 class, enable in BaseConfig This patch contains actions suggested in review comments at https://github.com/ansible/ansible-runner/pull/763 plus some more In display_callback/module.py, avoid modifying os.environ Remove parent class AWXDefaultCallbackModule, because it makes less since when there is not a job vs. adhoc class split As a consequence, only 1 parent callback class is needed the class which is the default, in absense of awx_display Move modifications of self.env from the runner module to the config module to be most consistent with the rest of the code base --- ansible_runner/config/_base.py | 11 +++ ansible_runner/config/runner.py | 7 -- ansible_runner/display_callback/module.py | 104 +++++++++------------- test/integration/test_config.py | 29 ++++++ test/unit/config/test__base.py | 1 + 5 files changed, 84 insertions(+), 68 deletions(-) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 30e1e1d38..493ed2b95 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -265,6 +265,17 @@ def _prepare_env(self, runner_mode='pexpect'): python_path += ':' self.env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(filter(None, (self.env.get('ANSIBLE_CALLBACK_PLUGINS'), callback_dir))) + if self.env.get('ANSIBLE_STDOUT_CALLBACK'): + # a custom stdout plugin should not be respected for adhoc command, unless load_callback_plugins set + if (not self.env.get('AD_HOC_COMMAND_ID')) or (not self.env.get('ANSIBLE_LOAD_CALLBACK_PLUGINS')): + self.env['ORIGINAL_STDOUT_CALLBACK'] = self.env.get('ANSIBLE_STDOUT_CALLBACK') + + if 'AD_HOC_COMMAND_ID' in self.env: + # force loading awx_display stdout callback for adhoc commands + self.env["ANSIBLE_LOAD_CALLBACK_PLUGINS"] = '1' + + self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' + self.env['ANSIBLE_RETRY_FILES_ENABLED'] = 'False' if 'ANSIBLE_HOST_KEY_CHECKING' not in self.env: self.env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' diff --git a/ansible_runner/config/runner.py b/ansible_runner/config/runner.py index e2d4a8ee6..c542e0b2d 100644 --- a/ansible_runner/config/runner.py +++ b/ansible_runner/config/runner.py @@ -243,13 +243,6 @@ def prepare_env(self): self.env["RUNNER_OMIT_EVENTS"] = str(self.omit_event_data) self.env["RUNNER_ONLY_FAILED_EVENTS"] = str(self.only_failed_event_data) - original_stdout_callback = os.getenv('ANSIBLE_STDOUT_CALLBACK') - if original_stdout_callback: - self.env['ORIGINAL_STDOUT_CALLBACK'] = original_stdout_callback - - self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' - self.env["ANSIBLE_LOAD_CALLBACK_PLUGINS"] = '1' - def prepare_command(self): try: cmdline_args = self.loader.load_file('args', string_types, encoding=None) diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/module.py index 7efd96ca9..f0a9f5dfc 100644 --- a/ansible_runner/display_callback/module.py +++ b/ansible_runner/display_callback/module.py @@ -28,29 +28,22 @@ # Ansible from ansible import constants as C -from ansible.plugins.callback import CallbackBase from ansible.plugins.loader import callback_loader from .events import event_context -user_specified_callback = os.getenv('ORIGINAL_STDOUT_CALLBACK') -runner_callback = os.environ.pop('ANSIBLE_STDOUT_CALLBACK', 'awx_display') - -if user_specified_callback: - os.environ['ANSIBLE_STDOUT_CALLBACK'] = user_specified_callback - -is_adhoc = os.getenv('AD_HOC_COMMAND_ID', False) +IS_ADHOC = os.getenv('AD_HOC_COMMAND_ID', False) # Dynamically construct base classes for our callback module, to support custom stdout callbacks. -if is_adhoc: +if os.getenv('ORIGINAL_STDOUT_CALLBACK'): + default_stdout_callback = os.getenv('ORIGINAL_STDOUT_CALLBACK') +elif IS_ADHOC: default_stdout_callback = 'minimal' else: - default_stdout_callback = C.config.get_config_value('DEFAULT_STDOUT_CALLBACK') + default_stdout_callback = 'default' DefaultCallbackModule = callback_loader.get(default_stdout_callback).__class__ -os.environ['ANSIBLE_STDOUT_CALLBACK'] = runner_callback - CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa @@ -58,11 +51,13 @@ def current_time(): return datetime.datetime.utcnow() -class BaseCallbackModule(CallbackBase): +class AWXCallbackModule(DefaultCallbackModule): ''' Callback module for logging ansible/ansible-playbook events. ''' + CALLBACK_NAME = 'awx_display' + CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'stdout' @@ -83,7 +78,7 @@ class BaseCallbackModule(CallbackBase): ] def __init__(self): - super(BaseCallbackModule, self).__init__() + super(AWXCallbackModule, self).__init__() self._host_start = {} self.task_uuids = set() self.duplicate_task_counts = collections.defaultdict(lambda: 1) @@ -188,7 +183,7 @@ def v2_playbook_on_start(self, playbook): uuid=self.playbook_uuid, ) with self.capture_event_data('playbook_on_start', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_start(playbook) + super(AWXCallbackModule, self).v2_playbook_on_start(playbook) def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, @@ -205,7 +200,7 @@ def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, unsafe=unsafe, ) with self.capture_event_data('playbook_on_vars_prompt', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_vars_prompt( + super(AWXCallbackModule, self).v2_playbook_on_vars_prompt( varname, private, prompt, encrypt, confirm, salt_size, salt, default, ) @@ -215,9 +210,11 @@ def v2_playbook_on_include(self, included_file): included_file=included_file._filename if included_file is not None else None, ) with self.capture_event_data('playbook_on_include', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_include(included_file) + super(AWXCallbackModule, self).v2_playbook_on_include(included_file) def v2_playbook_on_play_start(self, play): + if IS_ADHOC: + return play_uuid = str(play._uuid) if play_uuid in self.play_uuids: # When this play UUID repeats, it means the play is using the @@ -251,24 +248,27 @@ def v2_playbook_on_play_start(self, play): uuid=str(play._uuid), ) with self.capture_event_data('playbook_on_play_start', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_play_start(play) + super(AWXCallbackModule, self).v2_playbook_on_play_start(play) def v2_playbook_on_import_for_host(self, result, imported_file): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_import_for_host'): - super(BaseCallbackModule, self).v2_playbook_on_import_for_host(result, imported_file) + super(AWXCallbackModule, self).v2_playbook_on_import_for_host(result, imported_file) def v2_playbook_on_not_import_for_host(self, result, missing_file): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_not_import_for_host'): - super(BaseCallbackModule, self).v2_playbook_on_not_import_for_host(result, missing_file) + super(AWXCallbackModule, self).v2_playbook_on_not_import_for_host(result, missing_file) def v2_playbook_on_setup(self): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_setup'): - super(BaseCallbackModule, self).v2_playbook_on_setup() + super(AWXCallbackModule, self).v2_playbook_on_setup() def v2_playbook_on_task_start(self, task, is_conditional): + if IS_ADHOC: + self.set_task(task) + return # FIXME: Flag task path output as vv. task_uuid = str(task._uuid) if task_uuid in self.task_uuids: @@ -294,7 +294,7 @@ def v2_playbook_on_task_start(self, task, is_conditional): uuid=task_uuid, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_task_start(task, is_conditional) + super(AWXCallbackModule, self).v2_playbook_on_task_start(task, is_conditional) def v2_playbook_on_cleanup_task_start(self, task): # NOTE: Not used by Ansible 2.x. @@ -306,7 +306,7 @@ def v2_playbook_on_cleanup_task_start(self, task): is_conditional=True, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_cleanup_task_start(task) + super(AWXCallbackModule, self).v2_playbook_on_cleanup_task_start(task) def v2_playbook_on_handler_task_start(self, task): # NOTE: Re-using playbook_on_task_start event for this v2-specific @@ -320,15 +320,15 @@ def v2_playbook_on_handler_task_start(self, task): is_conditional=True, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_handler_task_start(task) + super(AWXCallbackModule, self).v2_playbook_on_handler_task_start(task) def v2_playbook_on_no_hosts_matched(self): with self.capture_event_data('playbook_on_no_hosts_matched'): - super(BaseCallbackModule, self).v2_playbook_on_no_hosts_matched() + super(AWXCallbackModule, self).v2_playbook_on_no_hosts_matched() def v2_playbook_on_no_hosts_remaining(self): with self.capture_event_data('playbook_on_no_hosts_remaining'): - super(BaseCallbackModule, self).v2_playbook_on_no_hosts_remaining() + super(AWXCallbackModule, self).v2_playbook_on_no_hosts_remaining() def v2_playbook_on_notify(self, handler, host): # NOTE: Not used by Ansible < 2.5. @@ -337,7 +337,7 @@ def v2_playbook_on_notify(self, handler, host): handler=handler.get_name(), ) with self.capture_event_data('playbook_on_notify', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_notify(handler, host) + super(AWXCallbackModule, self).v2_playbook_on_notify(handler, host) ''' ansible_stats is, retoractively, added in 2.2 @@ -358,7 +358,7 @@ def v2_playbook_on_stats(self, stats): ) with self.capture_event_data('playbook_on_stats', **event_data): - super(BaseCallbackModule, self).v2_playbook_on_stats(stats) + super(AWXCallbackModule, self).v2_playbook_on_stats(stats) @staticmethod def _get_event_loop(task): @@ -395,7 +395,7 @@ def v2_runner_on_ok(self, result): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_ok', **event_data): - super(BaseCallbackModule, self).v2_runner_on_ok(result) + super(AWXCallbackModule, self).v2_runner_on_ok(result) def v2_runner_on_failed(self, result, ignore_errors=False): # FIXME: Add verbosity for exception/results output. @@ -412,7 +412,7 @@ def v2_runner_on_failed(self, result, ignore_errors=False): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_failed', **event_data): - super(BaseCallbackModule, self).v2_runner_on_failed(result, ignore_errors) + super(AWXCallbackModule, self).v2_runner_on_failed(result, ignore_errors) def v2_runner_on_skipped(self, result): host_start, end_time, duration = self._get_result_timing_data(result) @@ -426,7 +426,7 @@ def v2_runner_on_skipped(self, result): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_skipped', **event_data): - super(BaseCallbackModule, self).v2_runner_on_skipped(result) + super(AWXCallbackModule, self).v2_runner_on_skipped(result) def v2_runner_on_unreachable(self, result): host_start, end_time, duration = self._get_result_timing_data(result) @@ -440,7 +440,7 @@ def v2_runner_on_unreachable(self, result): res=result._result, ) with self.capture_event_data('runner_on_unreachable', **event_data): - super(BaseCallbackModule, self).v2_runner_on_unreachable(result) + super(AWXCallbackModule, self).v2_runner_on_unreachable(result) def v2_runner_on_no_hosts(self, task): # NOTE: Not used by Ansible 2.x. @@ -448,7 +448,7 @@ def v2_runner_on_no_hosts(self, task): task=task, ) with self.capture_event_data('runner_on_no_hosts', **event_data): - super(BaseCallbackModule, self).v2_runner_on_no_hosts(task) + super(AWXCallbackModule, self).v2_runner_on_no_hosts(task) def v2_runner_on_async_poll(self, result): # NOTE: Not used by Ansible 2.x. @@ -459,7 +459,7 @@ def v2_runner_on_async_poll(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_poll', **event_data): - super(BaseCallbackModule, self).v2_runner_on_async_poll(result) + super(AWXCallbackModule, self).v2_runner_on_async_poll(result) def v2_runner_on_async_ok(self, result): # NOTE: Not used by Ansible 2.x. @@ -470,7 +470,7 @@ def v2_runner_on_async_ok(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_ok', **event_data): - super(BaseCallbackModule, self).v2_runner_on_async_ok(result) + super(AWXCallbackModule, self).v2_runner_on_async_ok(result) def v2_runner_on_async_failed(self, result): # NOTE: Not used by Ansible 2.x. @@ -481,7 +481,7 @@ def v2_runner_on_async_failed(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_failed', **event_data): - super(BaseCallbackModule, self).v2_runner_on_async_failed(result) + super(AWXCallbackModule, self).v2_runner_on_async_failed(result) def v2_runner_on_file_diff(self, result, diff): # NOTE: Not used by Ansible 2.x. @@ -491,7 +491,7 @@ def v2_runner_on_file_diff(self, result, diff): diff=diff, ) with self.capture_event_data('runner_on_file_diff', **event_data): - super(BaseCallbackModule, self).v2_runner_on_file_diff(result, diff) + super(AWXCallbackModule, self).v2_runner_on_file_diff(result, diff) def v2_on_file_diff(self, result): # NOTE: Logged as runner_on_file_diff. @@ -501,7 +501,7 @@ def v2_on_file_diff(self, result): diff=result._result.get('diff'), ) with self.capture_event_data('runner_on_file_diff', **event_data): - super(BaseCallbackModule, self).v2_on_file_diff(result) + super(AWXCallbackModule, self).v2_on_file_diff(result) def v2_runner_item_on_ok(self, result): event_data = dict( @@ -510,7 +510,7 @@ def v2_runner_item_on_ok(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_ok', **event_data): - super(BaseCallbackModule, self).v2_runner_item_on_ok(result) + super(AWXCallbackModule, self).v2_runner_item_on_ok(result) def v2_runner_item_on_failed(self, result): event_data = dict( @@ -519,7 +519,7 @@ def v2_runner_item_on_failed(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_failed', **event_data): - super(BaseCallbackModule, self).v2_runner_item_on_failed(result) + super(AWXCallbackModule, self).v2_runner_item_on_failed(result) def v2_runner_item_on_skipped(self, result): event_data = dict( @@ -528,7 +528,7 @@ def v2_runner_item_on_skipped(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_skipped', **event_data): - super(BaseCallbackModule, self).v2_runner_item_on_skipped(result) + super(AWXCallbackModule, self).v2_runner_item_on_skipped(result) def v2_runner_retry(self, result): event_data = dict( @@ -537,7 +537,7 @@ def v2_runner_retry(self, result): res=result._result, ) with self.capture_event_data('runner_retry', **event_data): - super(BaseCallbackModule, self).v2_runner_retry(result) + super(AWXCallbackModule, self).v2_runner_retry(result) def v2_runner_on_start(self, host, task): event_data = dict( @@ -546,22 +546,4 @@ def v2_runner_on_start(self, host, task): ) self._host_start[host.get_name()] = current_time() with self.capture_event_data('runner_on_start', **event_data): - super(BaseCallbackModule, self).v2_runner_on_start(host, task) - - -class AWXCallbackModule(BaseCallbackModule, DefaultCallbackModule): - - CALLBACK_NAME = 'awx_display' - - def v2_playbook_on_play_start(self, play): - if is_adhoc: - pass - else: - super().v2_playbook_on_play_start(play) - - - def v2_playbook_on_task_start(self, task, is_conditional): - if is_adhoc: - self.set_task(task) - else: - super().v2_playbook_on_task_start(task, is_conditional) + super(AWXCallbackModule, self).v2_runner_on_start(host, task) diff --git a/test/integration/test_config.py b/test/integration/test_config.py index b79eab16f..6cd90d04b 100644 --- a/test/integration/test_config.py +++ b/test/integration/test_config.py @@ -1,7 +1,36 @@ from ansible_runner.config._base import BaseConfig +from ansible_runner.interface import run + +import os def test_combine_python_and_file_settings(project_fixtures): rc = BaseConfig(private_data_dir=str(project_fixtures / 'job_env'), settings={'job_timeout': 40}) rc._prepare_env() assert rc.settings == {'job_timeout': 40, 'process_isolation': True} + + +def test_default_ansible_callback(project_fixtures): + """This is the reference case for stdout customization tests, assures default stdout callback is used""" + res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml') + stdout = res.stdout.read() + assert res.rc == 0, stdout + + assert '"msg": "Hello world!"' in stdout, stdout + + +def test_custom_stdout_callback_via_host_environ(project_fixtures, mocker): + mocker.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': 'yaml'}) + res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml') + stdout = res.stdout.read() + assert res.rc == 0, stdout + + assert 'msg: Hello world!' in stdout, stdout + + +def test_custom_stdout_callback_via_envvars(project_fixtures, mocker): + res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml', envvars={'ANSIBLE_STDOUT_CALLBACK': 'yaml'}) + stdout = res.stdout.read() + assert res.rc == 0, stdout + + assert 'msg: Hello world!' in stdout, stdout diff --git a/test/unit/config/test__base.py b/test/unit/config/test__base.py index 3c1c1eb37..04f3d86ee 100644 --- a/test/unit/config/test__base.py +++ b/test/unit/config/test__base.py @@ -191,6 +191,7 @@ def test_prepare_env_ansible_vars(mocker, tmp_path): assert not hasattr(rc, 'ssh_key_path') assert not hasattr(rc, 'command') + assert rc.env['ANSIBLE_STDOUT_CALLBACK'] == 'awx_display' assert rc.env['ANSIBLE_RETRY_FILES_ENABLED'] == 'False' assert rc.env['ANSIBLE_HOST_KEY_CHECKING'] == 'False' assert rc.env['AWX_ISOLATED_DATA_DIR'] == artifact_dir.joinpath(rc.ident).as_posix() From 837e83e825f40296c22953fbb5238e9ac2e55ccd Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 10 Jan 2022 23:02:19 -0500 Subject: [PATCH 04/14] Switch test to stdout callback in ansible.builtin --- test/integration/test_config.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/integration/test_config.py b/test/integration/test_config.py index 6cd90d04b..855014099 100644 --- a/test/integration/test_config.py +++ b/test/integration/test_config.py @@ -16,21 +16,24 @@ def test_default_ansible_callback(project_fixtures): stdout = res.stdout.read() assert res.rc == 0, stdout + assert 'ok: [host_1] => {' in stdout, stdout assert '"msg": "Hello world!"' in stdout, stdout def test_custom_stdout_callback_via_host_environ(project_fixtures, mocker): - mocker.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': 'yaml'}) + mocker.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': 'minimal'}) res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml') stdout = res.stdout.read() assert res.rc == 0, stdout - assert 'msg: Hello world!' in stdout, stdout + assert 'host_1 | SUCCESS => {' in stdout, stdout + assert '"msg": "Hello world!"' in stdout, stdout def test_custom_stdout_callback_via_envvars(project_fixtures, mocker): - res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml', envvars={'ANSIBLE_STDOUT_CALLBACK': 'yaml'}) + res = run(private_data_dir=str(project_fixtures / 'debug'), playbook='debug.yml', envvars={'ANSIBLE_STDOUT_CALLBACK': 'minimal'}) stdout = res.stdout.read() assert res.rc == 0, stdout - assert 'msg: Hello world!' in stdout, stdout + assert 'host_1 | SUCCESS => {' in stdout, stdout + assert '"msg": "Hello world!"' in stdout, stdout From f098e0031e149bd06519e3708b570f7d7ff2885e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 11 Jan 2022 14:53:15 -0500 Subject: [PATCH 05/14] Use correct metric for IS_ADHOC determination --- ansible_runner/config/_base.py | 11 +++++++---- test/integration/test_runner.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 493ed2b95..a74dedf87 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -265,14 +265,17 @@ def _prepare_env(self, runner_mode='pexpect'): python_path += ':' self.env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(filter(None, (self.env.get('ANSIBLE_CALLBACK_PLUGINS'), callback_dir))) + # this is an adhoc command if the module is specified, TODO: combine with logic in RunnerConfig class + is_adhoc = bool((self.binary is None) and (self.module is not None)) + if self.env.get('ANSIBLE_STDOUT_CALLBACK'): - # a custom stdout plugin should not be respected for adhoc command, unless load_callback_plugins set - if (not self.env.get('AD_HOC_COMMAND_ID')) or (not self.env.get('ANSIBLE_LOAD_CALLBACK_PLUGINS')): - self.env['ORIGINAL_STDOUT_CALLBACK'] = self.env.get('ANSIBLE_STDOUT_CALLBACK') + self.env['ORIGINAL_STDOUT_CALLBACK'] = self.env.get('ANSIBLE_STDOUT_CALLBACK') - if 'AD_HOC_COMMAND_ID' in self.env: + if is_adhoc: # force loading awx_display stdout callback for adhoc commands self.env["ANSIBLE_LOAD_CALLBACK_PLUGINS"] = '1' + if 'AD_HOC_COMMAND_ID' not in self.env: + self.env['AD_HOC_COMMAND_ID'] = '1' self.env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' diff --git a/test/integration/test_runner.py b/test/integration/test_runner.py index 627936c55..c3ccfa656 100644 --- a/test/integration/test_runner.py +++ b/test/integration/test_runner.py @@ -188,7 +188,7 @@ def test_run_command_ansible(rc): assert exitcode == 0 assert list(runner.events) != [] assert runner.stats != {} - assert list(runner.host_events('localhost')) != [] + assert list(runner.host_events('localhost')) != [], repr(list(runner.events)) stdout = runner.stdout assert stdout.read() != "" From 49026daf3b33207f1333b60e5657cb3863d53b57 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 11 Jan 2022 15:43:00 -0500 Subject: [PATCH 06/14] Accomidate other config classes --- ansible_runner/config/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index a74dedf87..37e5602a0 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -266,7 +266,7 @@ def _prepare_env(self, runner_mode='pexpect'): self.env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(filter(None, (self.env.get('ANSIBLE_CALLBACK_PLUGINS'), callback_dir))) # this is an adhoc command if the module is specified, TODO: combine with logic in RunnerConfig class - is_adhoc = bool((self.binary is None) and (self.module is not None)) + is_adhoc = bool((getattr(self, 'binary', None) is None) and (getattr(self, 'module', None) is not None)) if self.env.get('ANSIBLE_STDOUT_CALLBACK'): self.env['ORIGINAL_STDOUT_CALLBACK'] = self.env.get('ANSIBLE_STDOUT_CALLBACK') From d7eb76a6355db20d90e54f0dee9a4f0ba3b5bdf3 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 12 Jan 2022 09:06:40 -0500 Subject: [PATCH 07/14] Remove test hack --- test/integration/test_transmit_worker_process.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/test_transmit_worker_process.py b/test/integration/test_transmit_worker_process.py index fff45441d..cabc3c08e 100644 --- a/test/integration/test_transmit_worker_process.py +++ b/test/integration/test_transmit_worker_process.py @@ -30,8 +30,6 @@ def get_job_kwargs(self, job_type): job_kwargs = dict(module='setup', host_pattern='localhost') # also test use of user env vars job_kwargs['envvars'] = dict(MY_ENV_VAR='bogus') - if job_type == 'adhoc': - job_kwargs['envvars']['AD_HOC_COMMAND_ID'] = '1' return job_kwargs def check_artifacts(self, process_dir, job_type): From 70be623f7d8b6811825676bcfd2c61f9d714aa10 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 12 Jan 2022 14:18:51 -0500 Subject: [PATCH 08/14] Code move for single-file display callback --- ansible_runner/callbacks/__init__.py | 0 ansible_runner/callbacks/awx_display.py | 50 --- ansible_runner/display_callback/__init__.py | 24 -- ansible_runner/display_callback/display.py | 98 ------ ansible_runner/display_callback/events.py | 203 ------------ ansible_runner/display_callback/module.py | 342 +++++++++++++++++--- 6 files changed, 300 insertions(+), 417 deletions(-) delete mode 100644 ansible_runner/callbacks/__init__.py delete mode 100644 ansible_runner/callbacks/awx_display.py delete mode 100644 ansible_runner/display_callback/display.py delete mode 100644 ansible_runner/display_callback/events.py diff --git a/ansible_runner/callbacks/__init__.py b/ansible_runner/callbacks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ansible_runner/callbacks/awx_display.py b/ansible_runner/callbacks/awx_display.py deleted file mode 100644 index b66d18ca8..000000000 --- a/ansible_runner/callbacks/awx_display.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - - -DOCUMENTATION = ''' - callback: awx_display - short_description: Playbook event dispatcher for ansible-runner - version_added: "2.0" - description: - - This callback is necessary for ansible-runner to work - type: stdout - extends_documentation_fragment: - - default_callback - requirements: - - Set as stdout in config -''' - -# Python -import os # noqa -import sys # noqa - -# Add awx/lib to sys.path. -callback_lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -if callback_lib_path not in sys.path: - sys.path.insert(0, callback_lib_path) - -# Tower Display Callback -from display_callback import AWXCallbackModule # noqa - - -# In order to be recognized correctly, self.__class__.__name__ needs to -# match "CallbackModule" -class CallbackModule(AWXCallbackModule): - pass diff --git a/ansible_runner/display_callback/__init__.py b/ansible_runner/display_callback/__init__.py index 5f7e02469..e69de29bb 100644 --- a/ansible_runner/display_callback/__init__.py +++ b/ansible_runner/display_callback/__init__.py @@ -1,24 +0,0 @@ -# Copyright (c) 2016 Ansible by Red Hat, Inc. -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - -# AWX Display Callback -from . import display # noqa (wraps ansible.display.Display methods) -from .module import AWXCallbackModule - -__all__ = ['AWXCallbackModule'] diff --git a/ansible_runner/display_callback/display.py b/ansible_runner/display_callback/display.py deleted file mode 100644 index ad5e8ba37..000000000 --- a/ansible_runner/display_callback/display.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2016 Ansible by Red Hat, Inc. -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - -# Python -import functools -import sys -import uuid - -# Ansible -from ansible.utils.display import Display - -# Tower Display Callback -from .events import event_context - -__all__ = [] - - -def with_context(**context): - global event_context - - def wrap(f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - with event_context.set_local(**context): - return f(*args, **kwargs) - return wrapper - return wrap - - -for attr in dir(Display): - if attr.startswith('_') or 'cow' in attr or 'prompt' in attr: - continue - if attr in ('display', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv', 'verbose'): - continue - if not callable(getattr(Display, attr)): - continue - setattr(Display, attr, with_context(**{attr: True})(getattr(Display, attr))) - - -def with_verbosity(f): - global event_context - - @functools.wraps(f) - def wrapper(*args, **kwargs): - host = args[2] if len(args) >= 3 else kwargs.get('host', None) - caplevel = args[3] if len(args) >= 4 else kwargs.get('caplevel', 2) - context = dict(verbose=True, verbosity=(caplevel + 1)) - if host is not None: - context['remote_addr'] = host - with event_context.set_local(**context): - return f(*args, **kwargs) - return wrapper - - -Display.verbose = with_verbosity(Display.verbose) - - -def display_with_context(f): - - @functools.wraps(f) - def wrapper(*args, **kwargs): - log_only = args[5] if len(args) >= 6 else kwargs.get('log_only', False) - stderr = args[3] if len(args) >= 4 else kwargs.get('stderr', False) - event_uuid = event_context.get().get('uuid', None) - with event_context.display_lock: - # If writing only to a log file or there is already an event UUID - # set (from a callback module method), skip dumping the event data. - if log_only or event_uuid: - return f(*args, **kwargs) - try: - fileobj = sys.stderr if stderr else sys.stdout - event_context.add_local(uuid=str(uuid.uuid4())) - event_context.dump_begin(fileobj) - return f(*args, **kwargs) - finally: - event_context.dump_end(fileobj) - event_context.remove_local(uuid=None) - - return wrapper - - -Display.display = display_with_context(Display.display) diff --git a/ansible_runner/display_callback/events.py b/ansible_runner/display_callback/events.py deleted file mode 100644 index f3e067998..000000000 --- a/ansible_runner/display_callback/events.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) 2016 Ansible by Red Hat, Inc. -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from __future__ import (absolute_import, division, print_function) - -# Python -import base64 -import contextlib -import datetime -import json -import multiprocessing -import os -import stat -import threading -import uuid - -__all__ = ['event_context'] - - -# use a custom JSON serializer so we can properly handle !unsafe and !vault -# objects that may exist in events emitted by the callback plugin -# see: https://github.com/ansible/ansible/pull/38759 -class AnsibleJSONEncoderLocal(json.JSONEncoder): - ''' - The class AnsibleJSONEncoder exists in Ansible core for this function - this performs a mostly identical function via duck typing - ''' - - def default(self, o): - if getattr(o, 'yaml_tag', None) == '!vault': - encrypted_form = o._ciphertext - if isinstance(encrypted_form, bytes): - encrypted_form = encrypted_form.decode('utf-8') - return {'__ansible_vault': encrypted_form} - elif isinstance(o, (datetime.date, datetime.datetime)): - return o.isoformat() - return super(AnsibleJSONEncoderLocal, self).default(o) - - -class IsolatedFileWrite: - ''' - Class that will write partial event data to a file - ''' - - def __init__(self): - self.private_data_dir = os.getenv('AWX_ISOLATED_DATA_DIR') - - def set(self, key, value): - # Strip off the leading key identifying characters :1:ev- - event_uuid = key[len(':1:ev-'):] - # Write data in a staging area and then atomic move to pickup directory - filename = '{}-partial.json'.format(event_uuid) - if not os.path.exists(os.path.join(self.private_data_dir, 'job_events')): - os.mkdir(os.path.join(self.private_data_dir, 'job_events'), 0o700) - dropoff_location = os.path.join(self.private_data_dir, 'job_events', filename) - write_location = '.'.join([dropoff_location, 'tmp']) - partial_data = json.dumps(value, cls=AnsibleJSONEncoderLocal) - with os.fdopen(os.open(write_location, os.O_WRONLY | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR), 'w') as f: - f.write(partial_data) - os.rename(write_location, dropoff_location) - - -class EventContext(object): - ''' - Store global and local (per thread/process) data associated with callback - events and other display output methods. - ''' - - def __init__(self): - self.display_lock = multiprocessing.RLock() - self._local = threading.local() - if os.getenv('AWX_ISOLATED_DATA_DIR', False): - self.cache = IsolatedFileWrite() - - def add_local(self, **kwargs): - tls = vars(self._local) - ctx = tls.setdefault('_ctx', {}) - ctx.update(kwargs) - - def remove_local(self, **kwargs): - for key in kwargs.keys(): - self._local._ctx.pop(key, None) - - @contextlib.contextmanager - def set_local(self, **kwargs): - try: - self.add_local(**kwargs) - yield - finally: - self.remove_local(**kwargs) - - def get_local(self): - return getattr(getattr(self, '_local', None), '_ctx', {}) - - def add_global(self, **kwargs): - if not hasattr(self, '_global_ctx'): - self._global_ctx = {} - self._global_ctx.update(kwargs) - - def remove_global(self, **kwargs): - if hasattr(self, '_global_ctx'): - for key in kwargs.keys(): - self._global_ctx.pop(key, None) - - @contextlib.contextmanager - def set_global(self, **kwargs): - try: - self.add_global(**kwargs) - yield - finally: - self.remove_global(**kwargs) - - def get_global(self): - return getattr(self, '_global_ctx', {}) - - def get(self): - ctx = {} - ctx.update(self.get_global()) - ctx.update(self.get_local()) - return ctx - - def get_begin_dict(self): - omit_event_data = os.getenv("RUNNER_OMIT_EVENTS", "False").lower() == "true" - include_only_failed_event_data = os.getenv("RUNNER_ONLY_FAILED_EVENTS", "False").lower() == "true" - event_data = self.get() - event = event_data.pop('event', None) - if not event: - event = 'verbose' - for key in ('debug', 'verbose', 'deprecated', 'warning', 'system_warning', 'error'): - if event_data.get(key, False): - event = key - break - event_dict = dict(event=event) - should_process_event_data = (include_only_failed_event_data and event in ('runner_on_failed', 'runner_on_async_failed', 'runner_on_item_failed')) \ - or not include_only_failed_event_data - if os.getenv('JOB_ID', ''): - event_dict['job_id'] = int(os.getenv('JOB_ID', '0')) - if os.getenv('AD_HOC_COMMAND_ID', ''): - event_dict['ad_hoc_command_id'] = int(os.getenv('AD_HOC_COMMAND_ID', '0')) - if os.getenv('PROJECT_UPDATE_ID', ''): - event_dict['project_update_id'] = int(os.getenv('PROJECT_UPDATE_ID', '0')) - event_dict['pid'] = event_data.get('pid', os.getpid()) - event_dict['uuid'] = event_data.get('uuid', str(uuid.uuid4())) - event_dict['created'] = event_data.get('created', datetime.datetime.utcnow().isoformat()) - if not event_data.get('parent_uuid', None): - for key in ('task_uuid', 'play_uuid', 'playbook_uuid'): - parent_uuid = event_data.get(key, None) - if parent_uuid and parent_uuid != event_data.get('uuid', None): - event_dict['parent_uuid'] = parent_uuid - break - else: - event_dict['parent_uuid'] = event_data.get('parent_uuid', None) - if "verbosity" in event_data.keys(): - event_dict["verbosity"] = event_data.pop("verbosity") - if not omit_event_data and should_process_event_data: - max_res = int(os.getenv("MAX_EVENT_RES", 700000)) - if event not in ('playbook_on_stats',) and "res" in event_data and len(str(event_data['res'])) > max_res: - event_data['res'] = {} - else: - event_data = dict() - event_dict['event_data'] = event_data - return event_dict - - def get_end_dict(self): - return {} - - def dump(self, fileobj, data, max_width=78, flush=False): - b64data = base64.b64encode(json.dumps(data).encode('utf-8')).decode() - with self.display_lock: - # pattern corresponding to OutputEventFilter expectation - fileobj.write(u'\x1b[K') - for offset in range(0, len(b64data), max_width): - chunk = b64data[offset:offset + max_width] - escaped_chunk = u'{}\x1b[{}D'.format(chunk, len(chunk)) - fileobj.write(escaped_chunk) - fileobj.write(u'\x1b[K') - if flush: - fileobj.flush() - - def dump_begin(self, fileobj): - begin_dict = self.get_begin_dict() - self.cache.set(":1:ev-{}".format(begin_dict['uuid']), begin_dict) - self.dump(fileobj, {'uuid': begin_dict['uuid']}) - - def dump_end(self, fileobj): - self.dump(fileobj, self.get_end_dict(), flush=True) - - -event_context = EventContext() diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/module.py index f0a9f5dfc..6a5a68603 100644 --- a/ansible_runner/display_callback/module.py +++ b/ansible_runner/display_callback/module.py @@ -17,20 +17,39 @@ from __future__ import (absolute_import, division, print_function) + +DOCUMENTATION = ''' + callback: awx_display + short_description: Playbook event dispatcher for ansible-runner + version_added: "2.0" + description: + - This callback is necessary for ansible-runner to work + type: stdout + extends_documentation_fragment: + - default_callback + requirements: + - Set as stdout in config +''' + # Python -import collections -import contextlib -import datetime -import os -import sys -import uuid -from copy import copy +import json # noqa +import stat # noqa +import multiprocessing # noqa +import threading # noqa +import base64 # noqa +import functools # noqa +import collections # noqa +import contextlib # noqa +import datetime # noqa +import os # noqa +import sys # noqa +import uuid # noqa +from copy import copy # noqa # Ansible -from ansible import constants as C -from ansible.plugins.loader import callback_loader - -from .events import event_context +from ansible import constants as C # noqa +from ansible.plugins.loader import callback_loader # noqa +from ansible.utils.display import Display # noqa IS_ADHOC = os.getenv('AD_HOC_COMMAND_ID', False) @@ -51,7 +70,246 @@ def current_time(): return datetime.datetime.utcnow() -class AWXCallbackModule(DefaultCallbackModule): +# use a custom JSON serializer so we can properly handle !unsafe and !vault +# objects that may exist in events emitted by the callback plugin +# see: https://github.com/ansible/ansible/pull/38759 +class AnsibleJSONEncoderLocal(json.JSONEncoder): + ''' + The class AnsibleJSONEncoder exists in Ansible core for this function + this performs a mostly identical function via duck typing + ''' + + def default(self, o): + if getattr(o, 'yaml_tag', None) == '!vault': + encrypted_form = o._ciphertext + if isinstance(encrypted_form, bytes): + encrypted_form = encrypted_form.decode('utf-8') + return {'__ansible_vault': encrypted_form} + elif isinstance(o, (datetime.date, datetime.datetime)): + return o.isoformat() + return super(AnsibleJSONEncoderLocal, self).default(o) + + +class IsolatedFileWrite: + ''' + Class that will write partial event data to a file + ''' + + def __init__(self): + self.private_data_dir = os.getenv('AWX_ISOLATED_DATA_DIR') + + def set(self, key, value): + # Strip off the leading key identifying characters :1:ev- + event_uuid = key[len(':1:ev-'):] + # Write data in a staging area and then atomic move to pickup directory + filename = '{}-partial.json'.format(event_uuid) + if not os.path.exists(os.path.join(self.private_data_dir, 'job_events')): + os.mkdir(os.path.join(self.private_data_dir, 'job_events'), 0o700) + dropoff_location = os.path.join(self.private_data_dir, 'job_events', filename) + write_location = '.'.join([dropoff_location, 'tmp']) + partial_data = json.dumps(value, cls=AnsibleJSONEncoderLocal) + with os.fdopen(os.open(write_location, os.O_WRONLY | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR), 'w') as f: + f.write(partial_data) + os.rename(write_location, dropoff_location) + + +class EventContext(object): + ''' + Store global and local (per thread/process) data associated with callback + events and other display output methods. + ''' + + def __init__(self): + self.display_lock = multiprocessing.RLock() + self._local = threading.local() + if os.getenv('AWX_ISOLATED_DATA_DIR', False): + self.cache = IsolatedFileWrite() + + def add_local(self, **kwargs): + tls = vars(self._local) + ctx = tls.setdefault('_ctx', {}) + ctx.update(kwargs) + + def remove_local(self, **kwargs): + for key in kwargs.keys(): + self._local._ctx.pop(key, None) + + @contextlib.contextmanager + def set_local(self, **kwargs): + try: + self.add_local(**kwargs) + yield + finally: + self.remove_local(**kwargs) + + def get_local(self): + return getattr(getattr(self, '_local', None), '_ctx', {}) + + def add_global(self, **kwargs): + if not hasattr(self, '_global_ctx'): + self._global_ctx = {} + self._global_ctx.update(kwargs) + + def remove_global(self, **kwargs): + if hasattr(self, '_global_ctx'): + for key in kwargs.keys(): + self._global_ctx.pop(key, None) + + @contextlib.contextmanager + def set_global(self, **kwargs): + try: + self.add_global(**kwargs) + yield + finally: + self.remove_global(**kwargs) + + def get_global(self): + return getattr(self, '_global_ctx', {}) + + def get(self): + ctx = {} + ctx.update(self.get_global()) + ctx.update(self.get_local()) + return ctx + + def get_begin_dict(self): + omit_event_data = os.getenv("RUNNER_OMIT_EVENTS", "False").lower() == "true" + include_only_failed_event_data = os.getenv("RUNNER_ONLY_FAILED_EVENTS", "False").lower() == "true" + event_data = self.get() + event = event_data.pop('event', None) + if not event: + event = 'verbose' + for key in ('debug', 'verbose', 'deprecated', 'warning', 'system_warning', 'error'): + if event_data.get(key, False): + event = key + break + event_dict = dict(event=event) + should_process_event_data = (include_only_failed_event_data and event in ('runner_on_failed', 'runner_on_async_failed', 'runner_on_item_failed')) \ + or not include_only_failed_event_data + if os.getenv('JOB_ID', ''): + event_dict['job_id'] = int(os.getenv('JOB_ID', '0')) + if os.getenv('AD_HOC_COMMAND_ID', ''): + event_dict['ad_hoc_command_id'] = int(os.getenv('AD_HOC_COMMAND_ID', '0')) + if os.getenv('PROJECT_UPDATE_ID', ''): + event_dict['project_update_id'] = int(os.getenv('PROJECT_UPDATE_ID', '0')) + event_dict['pid'] = event_data.get('pid', os.getpid()) + event_dict['uuid'] = event_data.get('uuid', str(uuid.uuid4())) + event_dict['created'] = event_data.get('created', datetime.datetime.utcnow().isoformat()) + if not event_data.get('parent_uuid', None): + for key in ('task_uuid', 'play_uuid', 'playbook_uuid'): + parent_uuid = event_data.get(key, None) + if parent_uuid and parent_uuid != event_data.get('uuid', None): + event_dict['parent_uuid'] = parent_uuid + break + else: + event_dict['parent_uuid'] = event_data.get('parent_uuid', None) + if "verbosity" in event_data.keys(): + event_dict["verbosity"] = event_data.pop("verbosity") + if not omit_event_data and should_process_event_data: + max_res = int(os.getenv("MAX_EVENT_RES", 700000)) + if event not in ('playbook_on_stats',) and "res" in event_data and len(str(event_data['res'])) > max_res: + event_data['res'] = {} + else: + event_data = dict() + event_dict['event_data'] = event_data + return event_dict + + def get_end_dict(self): + return {} + + def dump(self, fileobj, data, max_width=78, flush=False): + b64data = base64.b64encode(json.dumps(data).encode('utf-8')).decode() + with self.display_lock: + # pattern corresponding to OutputEventFilter expectation + fileobj.write(u'\x1b[K') + for offset in range(0, len(b64data), max_width): + chunk = b64data[offset:offset + max_width] + escaped_chunk = u'{}\x1b[{}D'.format(chunk, len(chunk)) + fileobj.write(escaped_chunk) + fileobj.write(u'\x1b[K') + if flush: + fileobj.flush() + + def dump_begin(self, fileobj): + begin_dict = self.get_begin_dict() + self.cache.set(":1:ev-{}".format(begin_dict['uuid']), begin_dict) + self.dump(fileobj, {'uuid': begin_dict['uuid']}) + + def dump_end(self, fileobj): + self.dump(fileobj, self.get_end_dict(), flush=True) + + +event_context = EventContext() + + +def with_context(**context): + global event_context + + def wrap(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + with event_context.set_local(**context): + return f(*args, **kwargs) + return wrapper + return wrap + + +for attr in dir(Display): + if attr.startswith('_') or 'cow' in attr or 'prompt' in attr: + continue + if attr in ('display', 'v', 'vv', 'vvv', 'vvvv', 'vvvvv', 'vvvvvv', 'verbose'): + continue + if not callable(getattr(Display, attr)): + continue + setattr(Display, attr, with_context(**{attr: True})(getattr(Display, attr))) + + +def with_verbosity(f): + global event_context + + @functools.wraps(f) + def wrapper(*args, **kwargs): + host = args[2] if len(args) >= 3 else kwargs.get('host', None) + caplevel = args[3] if len(args) >= 4 else kwargs.get('caplevel', 2) + context = dict(verbose=True, verbosity=(caplevel + 1)) + if host is not None: + context['remote_addr'] = host + with event_context.set_local(**context): + return f(*args, **kwargs) + return wrapper + + +Display.verbose = with_verbosity(Display.verbose) + + +def display_with_context(f): + + @functools.wraps(f) + def wrapper(*args, **kwargs): + log_only = args[5] if len(args) >= 6 else kwargs.get('log_only', False) + stderr = args[3] if len(args) >= 4 else kwargs.get('stderr', False) + event_uuid = event_context.get().get('uuid', None) + with event_context.display_lock: + # If writing only to a log file or there is already an event UUID + # set (from a callback module method), skip dumping the event data. + if log_only or event_uuid: + return f(*args, **kwargs) + try: + fileobj = sys.stderr if stderr else sys.stdout + event_context.add_local(uuid=str(uuid.uuid4())) + event_context.dump_begin(fileobj) + return f(*args, **kwargs) + finally: + event_context.dump_end(fileobj) + event_context.remove_local(uuid=None) + + return wrapper + + +Display.display = display_with_context(Display.display) + + +class CallbackModule(DefaultCallbackModule): ''' Callback module for logging ansible/ansible-playbook events. ''' @@ -78,7 +336,7 @@ class AWXCallbackModule(DefaultCallbackModule): ] def __init__(self): - super(AWXCallbackModule, self).__init__() + super(CallbackModule, self).__init__() self._host_start = {} self.task_uuids = set() self.duplicate_task_counts = collections.defaultdict(lambda: 1) @@ -183,7 +441,7 @@ def v2_playbook_on_start(self, playbook): uuid=self.playbook_uuid, ) with self.capture_event_data('playbook_on_start', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_start(playbook) + super(CallbackModule, self).v2_playbook_on_start(playbook) def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, @@ -200,7 +458,7 @@ def v2_playbook_on_vars_prompt(self, varname, private=True, prompt=None, unsafe=unsafe, ) with self.capture_event_data('playbook_on_vars_prompt', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_vars_prompt( + super(CallbackModule, self).v2_playbook_on_vars_prompt( varname, private, prompt, encrypt, confirm, salt_size, salt, default, ) @@ -210,7 +468,7 @@ def v2_playbook_on_include(self, included_file): included_file=included_file._filename if included_file is not None else None, ) with self.capture_event_data('playbook_on_include', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_include(included_file) + super(CallbackModule, self).v2_playbook_on_include(included_file) def v2_playbook_on_play_start(self, play): if IS_ADHOC: @@ -248,22 +506,22 @@ def v2_playbook_on_play_start(self, play): uuid=str(play._uuid), ) with self.capture_event_data('playbook_on_play_start', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_play_start(play) + super(CallbackModule, self).v2_playbook_on_play_start(play) def v2_playbook_on_import_for_host(self, result, imported_file): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_import_for_host'): - super(AWXCallbackModule, self).v2_playbook_on_import_for_host(result, imported_file) + super(CallbackModule, self).v2_playbook_on_import_for_host(result, imported_file) def v2_playbook_on_not_import_for_host(self, result, missing_file): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_not_import_for_host'): - super(AWXCallbackModule, self).v2_playbook_on_not_import_for_host(result, missing_file) + super(CallbackModule, self).v2_playbook_on_not_import_for_host(result, missing_file) def v2_playbook_on_setup(self): # NOTE: Not used by Ansible 2.x. with self.capture_event_data('playbook_on_setup'): - super(AWXCallbackModule, self).v2_playbook_on_setup() + super(CallbackModule, self).v2_playbook_on_setup() def v2_playbook_on_task_start(self, task, is_conditional): if IS_ADHOC: @@ -294,7 +552,7 @@ def v2_playbook_on_task_start(self, task, is_conditional): uuid=task_uuid, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_task_start(task, is_conditional) + super(CallbackModule, self).v2_playbook_on_task_start(task, is_conditional) def v2_playbook_on_cleanup_task_start(self, task): # NOTE: Not used by Ansible 2.x. @@ -306,7 +564,7 @@ def v2_playbook_on_cleanup_task_start(self, task): is_conditional=True, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_cleanup_task_start(task) + super(CallbackModule, self).v2_playbook_on_cleanup_task_start(task) def v2_playbook_on_handler_task_start(self, task): # NOTE: Re-using playbook_on_task_start event for this v2-specific @@ -320,15 +578,15 @@ def v2_playbook_on_handler_task_start(self, task): is_conditional=True, ) with self.capture_event_data('playbook_on_task_start', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_handler_task_start(task) + super(CallbackModule, self).v2_playbook_on_handler_task_start(task) def v2_playbook_on_no_hosts_matched(self): with self.capture_event_data('playbook_on_no_hosts_matched'): - super(AWXCallbackModule, self).v2_playbook_on_no_hosts_matched() + super(CallbackModule, self).v2_playbook_on_no_hosts_matched() def v2_playbook_on_no_hosts_remaining(self): with self.capture_event_data('playbook_on_no_hosts_remaining'): - super(AWXCallbackModule, self).v2_playbook_on_no_hosts_remaining() + super(CallbackModule, self).v2_playbook_on_no_hosts_remaining() def v2_playbook_on_notify(self, handler, host): # NOTE: Not used by Ansible < 2.5. @@ -337,7 +595,7 @@ def v2_playbook_on_notify(self, handler, host): handler=handler.get_name(), ) with self.capture_event_data('playbook_on_notify', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_notify(handler, host) + super(CallbackModule, self).v2_playbook_on_notify(handler, host) ''' ansible_stats is, retoractively, added in 2.2 @@ -358,7 +616,7 @@ def v2_playbook_on_stats(self, stats): ) with self.capture_event_data('playbook_on_stats', **event_data): - super(AWXCallbackModule, self).v2_playbook_on_stats(stats) + super(CallbackModule, self).v2_playbook_on_stats(stats) @staticmethod def _get_event_loop(task): @@ -395,7 +653,7 @@ def v2_runner_on_ok(self, result): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_ok', **event_data): - super(AWXCallbackModule, self).v2_runner_on_ok(result) + super(CallbackModule, self).v2_runner_on_ok(result) def v2_runner_on_failed(self, result, ignore_errors=False): # FIXME: Add verbosity for exception/results output. @@ -412,7 +670,7 @@ def v2_runner_on_failed(self, result, ignore_errors=False): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_failed', **event_data): - super(AWXCallbackModule, self).v2_runner_on_failed(result, ignore_errors) + super(CallbackModule, self).v2_runner_on_failed(result, ignore_errors) def v2_runner_on_skipped(self, result): host_start, end_time, duration = self._get_result_timing_data(result) @@ -426,7 +684,7 @@ def v2_runner_on_skipped(self, result): event_loop=self._get_event_loop(result._task), ) with self.capture_event_data('runner_on_skipped', **event_data): - super(AWXCallbackModule, self).v2_runner_on_skipped(result) + super(CallbackModule, self).v2_runner_on_skipped(result) def v2_runner_on_unreachable(self, result): host_start, end_time, duration = self._get_result_timing_data(result) @@ -440,7 +698,7 @@ def v2_runner_on_unreachable(self, result): res=result._result, ) with self.capture_event_data('runner_on_unreachable', **event_data): - super(AWXCallbackModule, self).v2_runner_on_unreachable(result) + super(CallbackModule, self).v2_runner_on_unreachable(result) def v2_runner_on_no_hosts(self, task): # NOTE: Not used by Ansible 2.x. @@ -448,7 +706,7 @@ def v2_runner_on_no_hosts(self, task): task=task, ) with self.capture_event_data('runner_on_no_hosts', **event_data): - super(AWXCallbackModule, self).v2_runner_on_no_hosts(task) + super(CallbackModule, self).v2_runner_on_no_hosts(task) def v2_runner_on_async_poll(self, result): # NOTE: Not used by Ansible 2.x. @@ -459,7 +717,7 @@ def v2_runner_on_async_poll(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_poll', **event_data): - super(AWXCallbackModule, self).v2_runner_on_async_poll(result) + super(CallbackModule, self).v2_runner_on_async_poll(result) def v2_runner_on_async_ok(self, result): # NOTE: Not used by Ansible 2.x. @@ -470,7 +728,7 @@ def v2_runner_on_async_ok(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_ok', **event_data): - super(AWXCallbackModule, self).v2_runner_on_async_ok(result) + super(CallbackModule, self).v2_runner_on_async_ok(result) def v2_runner_on_async_failed(self, result): # NOTE: Not used by Ansible 2.x. @@ -481,7 +739,7 @@ def v2_runner_on_async_failed(self, result): jid=result._result.get('ansible_job_id'), ) with self.capture_event_data('runner_on_async_failed', **event_data): - super(AWXCallbackModule, self).v2_runner_on_async_failed(result) + super(CallbackModule, self).v2_runner_on_async_failed(result) def v2_runner_on_file_diff(self, result, diff): # NOTE: Not used by Ansible 2.x. @@ -491,7 +749,7 @@ def v2_runner_on_file_diff(self, result, diff): diff=diff, ) with self.capture_event_data('runner_on_file_diff', **event_data): - super(AWXCallbackModule, self).v2_runner_on_file_diff(result, diff) + super(CallbackModule, self).v2_runner_on_file_diff(result, diff) def v2_on_file_diff(self, result): # NOTE: Logged as runner_on_file_diff. @@ -501,7 +759,7 @@ def v2_on_file_diff(self, result): diff=result._result.get('diff'), ) with self.capture_event_data('runner_on_file_diff', **event_data): - super(AWXCallbackModule, self).v2_on_file_diff(result) + super(CallbackModule, self).v2_on_file_diff(result) def v2_runner_item_on_ok(self, result): event_data = dict( @@ -510,7 +768,7 @@ def v2_runner_item_on_ok(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_ok', **event_data): - super(AWXCallbackModule, self).v2_runner_item_on_ok(result) + super(CallbackModule, self).v2_runner_item_on_ok(result) def v2_runner_item_on_failed(self, result): event_data = dict( @@ -519,7 +777,7 @@ def v2_runner_item_on_failed(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_failed', **event_data): - super(AWXCallbackModule, self).v2_runner_item_on_failed(result) + super(CallbackModule, self).v2_runner_item_on_failed(result) def v2_runner_item_on_skipped(self, result): event_data = dict( @@ -528,7 +786,7 @@ def v2_runner_item_on_skipped(self, result): res=result._result, ) with self.capture_event_data('runner_item_on_skipped', **event_data): - super(AWXCallbackModule, self).v2_runner_item_on_skipped(result) + super(CallbackModule, self).v2_runner_item_on_skipped(result) def v2_runner_retry(self, result): event_data = dict( @@ -537,7 +795,7 @@ def v2_runner_retry(self, result): res=result._result, ) with self.capture_event_data('runner_retry', **event_data): - super(AWXCallbackModule, self).v2_runner_retry(result) + super(CallbackModule, self).v2_runner_retry(result) def v2_runner_on_start(self, host, task): event_data = dict( @@ -546,4 +804,4 @@ def v2_runner_on_start(self, host, task): ) self._host_start[host.get_name()] = current_time() with self.capture_event_data('runner_on_start', **event_data): - super(AWXCallbackModule, self).v2_runner_on_start(host, task) + super(CallbackModule, self).v2_runner_on_start(host, task) From ee15e0becdead3d7dd2ff2431ac6c951a199802b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 12 Jan 2022 14:24:36 -0500 Subject: [PATCH 09/14] Remove noqa comments with config change --- ansible_runner/display_callback/module.py | 34 +++++++++++------------ setup.cfg | 2 ++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/module.py index 6a5a68603..55e2bdc43 100644 --- a/ansible_runner/display_callback/module.py +++ b/ansible_runner/display_callback/module.py @@ -32,24 +32,24 @@ ''' # Python -import json # noqa -import stat # noqa -import multiprocessing # noqa -import threading # noqa -import base64 # noqa -import functools # noqa -import collections # noqa -import contextlib # noqa -import datetime # noqa -import os # noqa -import sys # noqa -import uuid # noqa -from copy import copy # noqa +import json +import stat +import multiprocessing +import threading +import base64 +import functools +import collections +import contextlib +import datetime +import os +import sys +import uuid +from copy import copy # Ansible -from ansible import constants as C # noqa -from ansible.plugins.loader import callback_loader # noqa -from ansible.utils.display import Display # noqa +from ansible import constants as C +from ansible.plugins.loader import callback_loader +from ansible.utils.display import Display IS_ADHOC = os.getenv('AD_HOC_COMMAND_ID', False) @@ -63,7 +63,7 @@ DefaultCallbackModule = callback_loader.get(default_stdout_callback).__class__ -CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa +CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" def current_time(): diff --git a/setup.cfg b/setup.cfg index 09ba02e7d..f3ad30a5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,5 @@ data-files = # W503 - Line break occurred before a binary operator ignore=W503 max-line-length=160 +per-file-ignores = + ansible_runner/display_callback/module.py:E402 From dd8349f044658c1e7be08278d6969c4df73736d8 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 12 Jan 2022 14:25:01 -0500 Subject: [PATCH 10/14] Move display callback to required name --- ansible_runner/config/_base.py | 2 -- .../display_callback/{ => callback}/__init__.py | 0 .../{module.py => callback/awx_display.py} | 0 setup.cfg | 2 +- utils/entrypoint.sh | 8 -------- 5 files changed, 1 insertion(+), 11 deletions(-) rename ansible_runner/display_callback/{ => callback}/__init__.py (100%) rename ansible_runner/display_callback/{module.py => callback/awx_display.py} (100%) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 37e5602a0..67402c41d 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -199,8 +199,6 @@ def _prepare_env(self, runner_mode='pexpect'): if self.containerized: self.container_name = "ansible_runner_{}".format(sanitize_container_name(self.ident)) self.env = {} - # Special flags to convey info to entrypoint or process in container - self.env['LAUNCHED_BY_RUNNER'] = '1' if self.process_isolation_executable == 'podman': # A kernel bug in RHEL < 8.5 causes podman to use the fuse-overlayfs driver. This results in errors when diff --git a/ansible_runner/display_callback/__init__.py b/ansible_runner/display_callback/callback/__init__.py similarity index 100% rename from ansible_runner/display_callback/__init__.py rename to ansible_runner/display_callback/callback/__init__.py diff --git a/ansible_runner/display_callback/module.py b/ansible_runner/display_callback/callback/awx_display.py similarity index 100% rename from ansible_runner/display_callback/module.py rename to ansible_runner/display_callback/callback/awx_display.py diff --git a/setup.cfg b/setup.cfg index f3ad30a5f..03dce1d38 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,4 +23,4 @@ data-files = ignore=W503 max-line-length=160 per-file-ignores = - ansible_runner/display_callback/module.py:E402 + ansible_runner/display_callback/awx_display.py:E402 diff --git a/utils/entrypoint.sh b/utils/entrypoint.sh index 2fb2bdcff..4cbf25dab 100755 --- a/utils/entrypoint.sh +++ b/utils/entrypoint.sh @@ -24,14 +24,6 @@ EOF fi -if [[ -n "${LAUNCHED_BY_RUNNER}" ]]; then - RUNNER_CALLBACKS=$(python3 -c "import ansible_runner.callbacks; print(ansible_runner.callbacks.__file__)") - - # TODO: respect user callback settings via - # env ANSIBLE_CALLBACK_PLUGINS or ansible.cfg - export ANSIBLE_CALLBACK_PLUGINS="$(dirname $RUNNER_CALLBACKS)" -fi - if [[ -d ${AWX_ISOLATED_DATA_DIR} ]]; then if output=$(ansible-galaxy collection list --format json 2> /dev/null); then echo $output > ${AWX_ISOLATED_DATA_DIR}/collections.json From 55d716cfe4fb82baef4a5d14d21c6c07a182891c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 12 Jan 2022 22:45:41 -0500 Subject: [PATCH 11/14] Code and test changes for callback volume mount Hook in stdout callback volume mount Share pdd and plugin mounts with the NONE base execution mode Remove PYTHONPATH assertions no longer relevant Attempt to patch up docs Refinement of docs page for the refactored display callback --- ansible_runner/config/_base.py | 23 ++++++++----------- .../display_callback/callback/awx_display.py | 4 ++++ ansible_runner/utils/__init__.py | 8 +++++++ docs/ansible_runner.callbacks.rst | 12 ---------- docs/ansible_runner.display_callback.rst | 22 +++--------------- docs/ansible_runner.rst | 1 - setup.cfg | 2 +- test/unit/config/test__base.py | 11 ++------- test/unit/config/test_ansible_cfg.py | 3 ++- test/unit/config/test_command.py | 2 ++ test/unit/config/test_doc.py | 4 +++- test/unit/config/test_inventory.py | 3 ++- test/unit/config/test_runner.py | 11 ++------- 13 files changed, 38 insertions(+), 68 deletions(-) delete mode 100644 docs/ansible_runner.callbacks.rst diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 67402c41d..556ace4c9 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -39,6 +39,8 @@ from ansible_runner.defaults import registry_auth_prefix from ansible_runner.loader import ArtifactLoader from ansible_runner.utils import ( + get_plugin_dir, + get_callback_dir, open_fifo_write, args2cmdline, sanitize_container_name, @@ -256,11 +258,7 @@ def _prepare_env(self, runner_mode='pexpect'): if not self.containerized: callback_dir = self.env.get('AWX_LIB_DIRECTORY', os.getenv('AWX_LIB_DIRECTORY')) if callback_dir is None: - callback_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], "..", "callbacks") - python_path = self.env.get('PYTHONPATH', os.getenv('PYTHONPATH', '')) - self.env['PYTHONPATH'] = ':'.join([python_path, callback_dir]) - if python_path and not python_path.endswith(':'): - python_path += ':' + callback_dir = get_callback_dir() self.env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(filter(None, (self.env.get('ANSIBLE_CALLBACK_PLUGINS'), callback_dir))) # this is an adhoc command if the module is specified, TODO: combine with logic in RunnerConfig class @@ -487,20 +485,17 @@ def wrap_args_for_containerization(self, args, execution_mode, cmdline_args): dst_mount_path="/runner/artifacts", labels=":Z") - # Mount the entire private_data_dir - # custom show paths inside private_data_dir do not make sense - self._update_volume_mount_paths(new_args, - "{}".format(self.private_data_dir), - dst_mount_path="/runner", - labels=":Z") else: subdir_path = os.path.join(self.private_data_dir, 'artifacts') if not os.path.exists(subdir_path): os.mkdir(subdir_path, 0o700) - # Mount the entire private_data_dir - # custom show paths inside private_data_dir do not make sense - self._update_volume_mount_paths(new_args, "{}".format(self.private_data_dir), dst_mount_path="/runner", labels=":Z") + # Mount the entire private_data_dir + # custom show paths inside private_data_dir do not make sense + self._update_volume_mount_paths(new_args, "{}".format(self.private_data_dir), dst_mount_path="/runner", labels=":Z") + + # Mount the stdout callback plugin from the ansible-runner code base + self._update_volume_mount_paths(new_args, "{}".format(get_plugin_dir()), dst_mount_path="/home/runner/.ansible/plugins", labels=":Z") if self.container_auth_data: # Pull in the necessary registry auth info, if there is a container cred diff --git a/ansible_runner/display_callback/callback/awx_display.py b/ansible_runner/display_callback/callback/awx_display.py index 55e2bdc43..0388a9a94 100644 --- a/ansible_runner/display_callback/callback/awx_display.py +++ b/ansible_runner/display_callback/callback/awx_display.py @@ -80,6 +80,10 @@ class AnsibleJSONEncoderLocal(json.JSONEncoder): ''' def default(self, o): + ''' + Returns JSON-valid representation for special Ansible python objects + which including vault objects and datetime objects + ''' if getattr(o, 'yaml_tag', None) == '!vault': encrypted_form = o._ciphertext if isinstance(encrypted_form, bytes): diff --git a/ansible_runner/utils/__init__.py b/ansible_runner/utils/__init__.py index 84b2b2a70..c0c64e815 100644 --- a/ansible_runner/utils/__init__.py +++ b/ansible_runner/utils/__init__.py @@ -46,6 +46,14 @@ def register_for_cleanup(folder): atexit.register(cleanup_folder, folder) +def get_plugin_dir(): + return os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "display_callback")) + + +def get_callback_dir(): + return os.path.join(get_plugin_dir(), 'callback') + + class Bunch(object): ''' diff --git a/docs/ansible_runner.callbacks.rst b/docs/ansible_runner.callbacks.rst deleted file mode 100644 index b2dd1787f..000000000 --- a/docs/ansible_runner.callbacks.rst +++ /dev/null @@ -1,12 +0,0 @@ -ansible_runner.callbacks package -================================ - -Submodules ----------- - -ansible_runner.callbacks.awx_display module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: ansible_runner.callbacks.awx_display - :members: - :undoc-members: diff --git a/docs/ansible_runner.display_callback.rst b/docs/ansible_runner.display_callback.rst index fd21c4dca..835ecf2a0 100644 --- a/docs/ansible_runner.display_callback.rst +++ b/docs/ansible_runner.display_callback.rst @@ -4,26 +4,10 @@ ansible_runner.display_callback package Submodules ---------- -ansible_runner.display_callback.display module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ansible_runner.display_callback.callback.awx_display module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. automodule:: ansible_runner.display_callback.display - :members: - :undoc-members: - :show-inheritance: - -ansible_runner.display_callback.events module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: ansible_runner.display_callback.events - :members: - :undoc-members: - :show-inheritance: - -ansible_runner.display_callback.module module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: ansible_runner.display_callback.module +.. automodule:: ansible_runner.display_callback.callback.awx_display :members: :undoc-members: diff --git a/docs/ansible_runner.rst b/docs/ansible_runner.rst index baf6391aa..cec321ae1 100644 --- a/docs/ansible_runner.rst +++ b/docs/ansible_runner.rst @@ -6,7 +6,6 @@ Subpackages .. toctree:: - ansible_runner.callbacks ansible_runner.config ansible_runner.display_callback diff --git a/setup.cfg b/setup.cfg index 03dce1d38..a1a7cca53 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,4 +23,4 @@ data-files = ignore=W503 max-line-length=160 per-file-ignores = - ansible_runner/display_callback/awx_display.py:E402 + ansible_runner/display_callback/callback/awx_display.py:E402 diff --git a/test/unit/config/test__base.py b/test/unit/config/test__base.py index 04f3d86ee..594e2266c 100644 --- a/test/unit/config/test__base.py +++ b/test/unit/config/test__base.py @@ -13,6 +13,7 @@ from ansible_runner.config._base import BaseConfig, BaseExecutionMode from ansible_runner.loader import ArtifactLoader from ansible_runner.exceptions import ConfigurationError +from ansible_runner.utils import get_plugin_dir try: Pattern = re._pattern_type @@ -176,7 +177,6 @@ def test_prepare_env_defaults(): def test_prepare_env_ansible_vars(mocker, tmp_path): mocker.patch.dict('os.environ', { - 'PYTHONPATH': '/python_path_via_environ', 'AWX_LIB_DIRECTORY': '/awx_lib_directory_via_environ', }) @@ -195,14 +195,6 @@ def test_prepare_env_ansible_vars(mocker, tmp_path): assert rc.env['ANSIBLE_RETRY_FILES_ENABLED'] == 'False' assert rc.env['ANSIBLE_HOST_KEY_CHECKING'] == 'False' assert rc.env['AWX_ISOLATED_DATA_DIR'] == artifact_dir.joinpath(rc.ident).as_posix() - assert rc.env['PYTHONPATH'] == '/python_path_via_environ:/awx_lib_directory_via_environ', \ - "PYTHONPATH is the union of the env PYTHONPATH and AWX_LIB_DIRECTORY" - - del rc.env['PYTHONPATH'] - os.environ['PYTHONPATH'] = "/foo/bar/python_path_via_environ" - rc._prepare_env() - assert rc.env['PYTHONPATH'] == "/foo/bar/python_path_via_environ:/awx_lib_directory_via_environ", \ - "PYTHONPATH is the union of the explicit env['PYTHONPATH'] override and AWX_LIB_DIRECTORY" def test_prepare_with_ssh_key(mocker, tmp_path): @@ -325,6 +317,7 @@ def test_containerization_settings(tmp_path, runtime, mocker): expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_ansible_cfg.py b/test/unit/config/test_ansible_cfg.py index ecc96bea7..7d8302790 100644 --- a/test/unit/config/test_ansible_cfg.py +++ b/test/unit/config/test_ansible_cfg.py @@ -6,7 +6,7 @@ from ansible_runner.config.ansible_cfg import AnsibleCfgConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path +from ansible_runner.utils import get_executable_path, get_plugin_dir def test_ansible_cfg_init_defaults(tmp_path, patch_private_data_dir): @@ -91,6 +91,7 @@ def test_prepare_config_command_with_containerization(tmp_path, runtime, mocker) expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_command.py b/test/unit/config/test_command.py index 75bc2db64..6fc49afca 100644 --- a/test/unit/config/test_command.py +++ b/test/unit/config/test_command.py @@ -6,6 +6,7 @@ from ansible_runner.config.command import CommandConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError +from ansible_runner.utils import get_plugin_dir def test_ansible_config_defaults(tmp_path, patch_private_data_dir): @@ -105,6 +106,7 @@ def test_prepare_run_command_with_containerization(tmp_path, runtime, mocker): expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_doc.py b/test/unit/config/test_doc.py index 1058dac19..0f9a620a1 100755 --- a/test/unit/config/test_doc.py +++ b/test/unit/config/test_doc.py @@ -6,7 +6,7 @@ from ansible_runner.config.doc import DocConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path +from ansible_runner.utils import get_executable_path, get_plugin_dir def test_ansible_doc_defaults(tmp_path, patch_private_data_dir): @@ -101,6 +101,7 @@ def test_prepare_plugin_docs_command_with_containerization(tmp_path, runtime, mo expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) @@ -169,6 +170,7 @@ def test_prepare_plugin_list_command_with_containerization(tmp_path, runtime, mo expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_inventory.py b/test/unit/config/test_inventory.py index 65fbe5d34..14d0ff684 100644 --- a/test/unit/config/test_inventory.py +++ b/test/unit/config/test_inventory.py @@ -6,7 +6,7 @@ from ansible_runner.config.inventory import InventoryConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path +from ansible_runner.utils import get_executable_path, get_plugin_dir def test_ansible_inventory_init_defaults(tmp_path, patch_private_data_dir): @@ -126,6 +126,7 @@ def test_prepare_inventory_command_with_containerization(tmp_path, runtime, mock expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), + '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_runner.py b/test/unit/config/test_runner.py index 2940712fa..44b96b8f9 100644 --- a/test/unit/config/test_runner.py +++ b/test/unit/config/test_runner.py @@ -14,6 +14,7 @@ from ansible_runner.interface import init_runner from ansible_runner.loader import ArtifactLoader from ansible_runner.exceptions import ConfigurationError +from ansible_runner.utils import get_plugin_dir try: Pattern = re._pattern_type @@ -405,7 +406,6 @@ def test_prepare_with_defaults(mocker): def test_prepare(mocker): mocker.patch.dict('os.environ', { - 'PYTHONPATH': '/python_path_via_environ', 'AWX_LIB_DIRECTORY': '/awx_lib_directory_via_environ', }) mocker.patch('os.makedirs', return_value=True) @@ -431,14 +431,6 @@ def test_prepare(mocker): assert rc.env['ANSIBLE_RETRY_FILES_ENABLED'] == 'False' assert rc.env['ANSIBLE_HOST_KEY_CHECKING'] == 'False' assert rc.env['AWX_ISOLATED_DATA_DIR'] == '/' - assert rc.env['PYTHONPATH'] == '/python_path_via_environ:/awx_lib_directory_via_environ', \ - "PYTHONPATH is the union of the env PYTHONPATH and AWX_LIB_DIRECTORY" - - del rc.env['PYTHONPATH'] - os.environ['PYTHONPATH'] = "/foo/bar/python_path_via_environ" - rc.prepare() - assert rc.env['PYTHONPATH'] == "/foo/bar/python_path_via_environ:/awx_lib_directory_via_environ", \ - "PYTHONPATH is the union of the explicit env['PYTHONPATH'] override and AWX_LIB_DIRECTORY" def test_prepare_with_ssh_key(mocker): @@ -740,6 +732,7 @@ def test_containerization_settings(tmp_path, runtime, mocker): expected_command_start = [runtime, 'run', '--rm', '--tty', '--interactive', '--workdir', '/runner/project'] + \ ['-v', '{}/:/runner/:Z'.format(rc.private_data_dir)] + \ + ['-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir())] + \ ['-v', '/host1/:/container1/', '-v', '/host2/:/container2/'] + \ ['--env-file', '{}/env.list'.format(rc.artifact_dir)] + \ extra_container_args + \ From 851f4d369f7cf035c3cf671fa0dc5cdbfeb91710 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 17 Jan 2022 09:34:01 -0500 Subject: [PATCH 12/14] Change mount location and introduce shared method This changes the location of the mount for the stdout callback prior value was at /home/runner/.ansible/plugins new mount location is at /home/runner/.ansible/plugins/callback This is more targeted and more desirable for this reason. We cannot mount single-files like awx_display.py, due to permissions error, which may be because changes are needed in the Dockerfile or some other reason To carry this out, new utils method callback_mount is introduced to make such changes easier to carry out in the future --- ansible_runner/config/_base.py | 4 ++-- ansible_runner/utils/__init__.py | 12 ++++++++++++ test/unit/config/test__base.py | 4 ++-- test/unit/config/test_ansible_cfg.py | 4 ++-- test/unit/config/test_command.py | 4 ++-- test/unit/config/test_doc.py | 6 +++--- test/unit/config/test_inventory.py | 4 ++-- test/unit/config/test_runner.py | 4 ++-- 8 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index 556ace4c9..e62c32a78 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -39,7 +39,7 @@ from ansible_runner.defaults import registry_auth_prefix from ansible_runner.loader import ArtifactLoader from ansible_runner.utils import ( - get_plugin_dir, + callback_mount, get_callback_dir, open_fifo_write, args2cmdline, @@ -495,7 +495,7 @@ def wrap_args_for_containerization(self, args, execution_mode, cmdline_args): self._update_volume_mount_paths(new_args, "{}".format(self.private_data_dir), dst_mount_path="/runner", labels=":Z") # Mount the stdout callback plugin from the ansible-runner code base - self._update_volume_mount_paths(new_args, "{}".format(get_plugin_dir()), dst_mount_path="/home/runner/.ansible/plugins", labels=":Z") + self._update_volume_mount_paths(new_args, callback_mount()[0], dst_mount_path=callback_mount()[1], labels=":Z") if self.container_auth_data: # Pull in the necessary registry auth info, if there is a container cred diff --git a/ansible_runner/utils/__init__.py b/ansible_runner/utils/__init__.py index c0c64e815..9436dfc9f 100644 --- a/ansible_runner/utils/__init__.py +++ b/ansible_runner/utils/__init__.py @@ -54,6 +54,18 @@ def get_callback_dir(): return os.path.join(get_plugin_dir(), 'callback') +def callback_mount(): + ''' + Return a tuple that gives mount points for the standard out callback + in the form of (, ) + ''' + container_dot_ansible = '/home/runner/.ansible' + rel_path = ('callback', '',) + callback_file = os.path.join(get_plugin_dir(), *rel_path) + container_path = os.path.join(container_dot_ansible, 'plugins', *rel_path) + return (callback_file, container_path) + + class Bunch(object): ''' diff --git a/test/unit/config/test__base.py b/test/unit/config/test__base.py index 594e2266c..5c2c8c407 100644 --- a/test/unit/config/test__base.py +++ b/test/unit/config/test__base.py @@ -13,7 +13,7 @@ from ansible_runner.config._base import BaseConfig, BaseExecutionMode from ansible_runner.loader import ArtifactLoader from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_plugin_dir +from ansible_runner.utils import callback_mount try: Pattern = re._pattern_type @@ -317,7 +317,7 @@ def test_containerization_settings(tmp_path, runtime, mocker): expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_ansible_cfg.py b/test/unit/config/test_ansible_cfg.py index 7d8302790..da7e7e0cb 100644 --- a/test/unit/config/test_ansible_cfg.py +++ b/test/unit/config/test_ansible_cfg.py @@ -6,7 +6,7 @@ from ansible_runner.config.ansible_cfg import AnsibleCfgConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path, get_plugin_dir +from ansible_runner.utils import get_executable_path, callback_mount def test_ansible_cfg_init_defaults(tmp_path, patch_private_data_dir): @@ -91,7 +91,7 @@ def test_prepare_config_command_with_containerization(tmp_path, runtime, mocker) expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_command.py b/test/unit/config/test_command.py index 6fc49afca..5c2071ea6 100644 --- a/test/unit/config/test_command.py +++ b/test/unit/config/test_command.py @@ -6,7 +6,7 @@ from ansible_runner.config.command import CommandConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_plugin_dir +from ansible_runner.utils import callback_mount def test_ansible_config_defaults(tmp_path, patch_private_data_dir): @@ -106,7 +106,7 @@ def test_prepare_run_command_with_containerization(tmp_path, runtime, mocker): expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_doc.py b/test/unit/config/test_doc.py index 0f9a620a1..a1dfd24f1 100755 --- a/test/unit/config/test_doc.py +++ b/test/unit/config/test_doc.py @@ -6,7 +6,7 @@ from ansible_runner.config.doc import DocConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path, get_plugin_dir +from ansible_runner.utils import get_executable_path, callback_mount def test_ansible_doc_defaults(tmp_path, patch_private_data_dir): @@ -101,7 +101,7 @@ def test_prepare_plugin_docs_command_with_containerization(tmp_path, runtime, mo expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) @@ -170,7 +170,7 @@ def test_prepare_plugin_list_command_with_containerization(tmp_path, runtime, mo expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_inventory.py b/test/unit/config/test_inventory.py index 14d0ff684..dd81027f9 100644 --- a/test/unit/config/test_inventory.py +++ b/test/unit/config/test_inventory.py @@ -6,7 +6,7 @@ from ansible_runner.config.inventory import InventoryConfig from ansible_runner.config._base import BaseExecutionMode from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_executable_path, get_plugin_dir +from ansible_runner.utils import get_executable_path, callback_mount def test_ansible_inventory_init_defaults(tmp_path, patch_private_data_dir): @@ -126,7 +126,7 @@ def test_prepare_inventory_command_with_containerization(tmp_path, runtime, mock expected_command_start.extend([ '-v', '{}/artifacts/:/runner/artifacts/:Z'.format(rc.private_data_dir), '-v', '{}/:/runner/:Z'.format(rc.private_data_dir), - '-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir()), + '-v', '{0}:{1}:Z'.format(*callback_mount()), '--env-file', '{}/env.list'.format(rc.artifact_dir), ]) diff --git a/test/unit/config/test_runner.py b/test/unit/config/test_runner.py index 44b96b8f9..725a7fde5 100644 --- a/test/unit/config/test_runner.py +++ b/test/unit/config/test_runner.py @@ -14,7 +14,7 @@ from ansible_runner.interface import init_runner from ansible_runner.loader import ArtifactLoader from ansible_runner.exceptions import ConfigurationError -from ansible_runner.utils import get_plugin_dir +from ansible_runner.utils import callback_mount try: Pattern = re._pattern_type @@ -732,7 +732,7 @@ def test_containerization_settings(tmp_path, runtime, mocker): expected_command_start = [runtime, 'run', '--rm', '--tty', '--interactive', '--workdir', '/runner/project'] + \ ['-v', '{}/:/runner/:Z'.format(rc.private_data_dir)] + \ - ['-v', '{}/:/home/runner/.ansible/plugins/:Z'.format(get_plugin_dir())] + \ + ['-v', '{0}:{1}:Z'.format(*callback_mount())] + \ ['-v', '/host1/:/container1/', '-v', '/host2/:/container2/'] + \ ['--env-file', '{}/env.list'.format(rc.artifact_dir)] + \ extra_container_args + \ From df44cea9fea45fbf4bfe5c252d6a5dcc2e931f7e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 26 Jan 2022 22:32:24 -0500 Subject: [PATCH 13/14] Workaround when install is not owned by current user --- ansible_runner/config/_base.py | 3 ++- ansible_runner/utils/__init__.py | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible_runner/config/_base.py b/ansible_runner/config/_base.py index e62c32a78..17c822572 100644 --- a/ansible_runner/config/_base.py +++ b/ansible_runner/config/_base.py @@ -495,7 +495,8 @@ def wrap_args_for_containerization(self, args, execution_mode, cmdline_args): self._update_volume_mount_paths(new_args, "{}".format(self.private_data_dir), dst_mount_path="/runner", labels=":Z") # Mount the stdout callback plugin from the ansible-runner code base - self._update_volume_mount_paths(new_args, callback_mount()[0], dst_mount_path=callback_mount()[1], labels=":Z") + mount_paths = callback_mount(copy_if_needed=True) + self._update_volume_mount_paths(new_args, mount_paths[0], dst_mount_path=mount_paths[1], labels=":Z") if self.container_auth_data: # Pull in the necessary registry auth info, if there is a container cred diff --git a/ansible_runner/utils/__init__.py b/ansible_runner/utils/__init__.py index 9436dfc9f..e295fd5fd 100644 --- a/ansible_runner/utils/__init__.py +++ b/ansible_runner/utils/__init__.py @@ -11,6 +11,8 @@ import subprocess import base64 import threading +from pathlib import Path +import pwd import pipes import uuid import codecs @@ -54,16 +56,27 @@ def get_callback_dir(): return os.path.join(get_plugin_dir(), 'callback') -def callback_mount(): +def callback_mount(copy_if_needed=False): ''' Return a tuple that gives mount points for the standard out callback in the form of (, ) + if copy_if_needed is set, and the install is owned by another user, + it will copy the plugin to a tmpdir for the mount in anticipation of SELinux problems ''' container_dot_ansible = '/home/runner/.ansible' rel_path = ('callback', '',) - callback_file = os.path.join(get_plugin_dir(), *rel_path) + host_path = os.path.join(get_plugin_dir(), *rel_path) + if copy_if_needed: + current_user = pwd.getpwuid(os.geteuid()).pw_name + callback_dir = get_callback_dir() + callback_owner = Path(callback_dir).owner() + if current_user != callback_owner: + tmp_path = tempfile.mkdtemp(prefix='ansible_runner_plugins_') + host_path = os.path.join(tmp_path, 'callback') + register_for_cleanup(host_path) + shutil.copytree(callback_dir, host_path) container_path = os.path.join(container_dot_ansible, 'plugins', *rel_path) - return (callback_file, container_path) + return (host_path, container_path) class Bunch(object): From 3c3a461158ada046ab272fdb07a1073f1c9d1348 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 27 Jan 2022 20:23:02 -0500 Subject: [PATCH 14/14] Add test to simulate non-root container run --- ansible_runner/utils/__init__.py | 13 +++++++++---- .../test_container_management.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/ansible_runner/utils/__init__.py b/ansible_runner/utils/__init__.py index e295fd5fd..251ea5d3b 100644 --- a/ansible_runner/utils/__init__.py +++ b/ansible_runner/utils/__init__.py @@ -56,6 +56,13 @@ def get_callback_dir(): return os.path.join(get_plugin_dir(), 'callback') +def is_dir_owner(directory): + '''Returns True if current user is the owner of directory''' + current_user = pwd.getpwuid(os.geteuid()).pw_name + callback_owner = Path(directory).owner() + return bool(current_user == callback_owner) + + def callback_mount(copy_if_needed=False): ''' Return a tuple that gives mount points for the standard out callback @@ -67,13 +74,11 @@ def callback_mount(copy_if_needed=False): rel_path = ('callback', '',) host_path = os.path.join(get_plugin_dir(), *rel_path) if copy_if_needed: - current_user = pwd.getpwuid(os.geteuid()).pw_name callback_dir = get_callback_dir() - callback_owner = Path(callback_dir).owner() - if current_user != callback_owner: + if not is_dir_owner(callback_dir): tmp_path = tempfile.mkdtemp(prefix='ansible_runner_plugins_') + register_for_cleanup(tmp_path) host_path = os.path.join(tmp_path, 'callback') - register_for_cleanup(host_path) shutil.copytree(callback_dir, host_path) container_path = os.path.join(container_dot_ansible, 'plugins', *rel_path) return (host_path, container_path) diff --git a/test/integration/containerized/test_container_management.py b/test/integration/containerized/test_container_management.py index c2a6316a6..8d71a972a 100644 --- a/test/integration/containerized/test_container_management.py +++ b/test/integration/containerized/test_container_management.py @@ -75,6 +75,25 @@ def test_cancel_will_remove_container(project_fixtures, runtime, cli): ), 'Found a running container, they should have all been stopped' +@pytest.mark.test_all_runtimes +def test_non_owner_install(mocker, project_fixtures, runtime): + """Simulates a run on a conputer where ansible-runner install is not owned by current user""" + mocker.patch('ansible_runner.utils.is_dir_owner', return_value=False) + + private_data_dir = project_fixtures / 'debug' + res = run( + private_data_dir=private_data_dir, + playbook='debug.yml', + settings={ + 'process_isolation_executable': runtime, + 'process_isolation': True + } + ) + stdout = res.stdout.read() + assert res.rc == 0, stdout + assert res.status == 'successful' + + @pytest.mark.test_all_runtimes def test_invalid_registry_host(tmp_path, runtime): pdd_path = tmp_path / 'private_data_dir'