From 9c5342d3fbb87d30c8c599c029593cf4da7024af Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 19 Jan 2022 16:01:59 -0500 Subject: [PATCH 1/3] Preserve timestamps when unstreaming dirs --- ansible_runner/utils/streaming.py | 6 +++++ test/unit/utils/test_utils.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/ansible_runner/utils/streaming.py b/ansible_runner/utils/streaming.py index 1fa6f687c..53f0f614c 100644 --- a/ansible_runner/utils/streaming.py +++ b/ansible_runner/utils/streaming.py @@ -1,3 +1,4 @@ +import time import tempfile import zipfile import os @@ -85,6 +86,11 @@ def unstream_dir(stream, length, target_directory): archive.extract(info.filename, path=target_directory) + # Fancy logic to preserve modification times + # https://stackoverflow.com/questions/9813243/extract-files-from-zip-file-and-retain-mod-date + date_time = time.mktime(info.date_time + (0, 0, -1)) + os.utime(out_path, times=(date_time, date_time)) + if is_symlink: link = open(out_path).read() os.remove(out_path) diff --git a/test/unit/utils/test_utils.py b/test/unit/utils/test_utils.py index faaf7ea0f..740cfa851 100644 --- a/test/unit/utils/test_utils.py +++ b/test/unit/utils/test_utils.py @@ -1,7 +1,9 @@ +import datetime import io import json import os import signal +import time from pathlib import Path @@ -147,6 +149,48 @@ def test_transmit_permissions(tmp_path, fperm): assert oct(new_file_path.stat().st_mode) == oct(old_file_path.stat().st_mode) +def test_transmit_modtimes(tmp_path): + source_dir = tmp_path / 'source' + source_dir.mkdir() + + (source_dir / 'b.txt').touch() + time.sleep(2.0) # flaky for anything less because of integer math + (source_dir / 'a.txt').touch() + + very_old_file = source_dir / 'very_old.txt' + very_old_file.touch() + old_datetime = os.path.getmtime(source_dir / 'a.txt') - datetime.timedelta(days=1).total_seconds() + os.utime(very_old_file, (old_datetime, old_datetime)) + + # sanity, verify assertions pass for source dir + mod_delta = os.path.getmtime(source_dir / 'a.txt') - os.path.getmtime(source_dir / 'b.txt') + assert mod_delta >= 1.0 + + outgoing_buffer = io.BytesIO() + outgoing_buffer.name = 'not_stdout' + stream_dir(source_dir, outgoing_buffer) + + dest_dir = tmp_path / 'dest' + dest_dir.mkdir() + + outgoing_buffer.seek(0) + first_line = outgoing_buffer.readline() + size_data = json.loads(first_line.strip()) + unstream_dir(outgoing_buffer, size_data['zipfile'], dest_dir) + + # Assure modification times are internally consistent + mod_delta = os.path.getmtime(dest_dir / 'a.txt') - os.path.getmtime(dest_dir / 'b.txt') + assert mod_delta >= 1.0 + + # Assure modification times are same as original + for filename in ('a.txt', 'b.txt'): + assert os.path.getmtime(dest_dir / filename) == os.path.getmtime(dest_dir / filename) + + # Assure the very old timestamp is preserved + old_delta = os.path.getmtime(dest_dir / 'a.txt') - os.path.getmtime(dest_dir / 'very_old.txt') + assert old_delta >= datetime.timedelta(days=1).total_seconds() - 2. + + def test_signal_handler(mocker): """Test the default handler is set to handle the correct signals""" From a6a6b94ad6310f17f36a942bf40e91bd9316636b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 25 Jan 2022 15:45:18 -0500 Subject: [PATCH 2/3] Detail AWX connection in brief comments --- ansible_runner/utils/streaming.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible_runner/utils/streaming.py b/ansible_runner/utils/streaming.py index 53f0f614c..b8812c93d 100644 --- a/ansible_runner/utils/streaming.py +++ b/ansible_runner/utils/streaming.py @@ -69,6 +69,7 @@ def unstream_dir(stream, length, target_directory): with zipfile.ZipFile(tmp.name, "r") as archive: # Fancy extraction in order to preserve permissions + # AWX relies on the execution bit, in particular, for inventory # https://www.burgundywall.com/post/preserving-file-perms-with-python-zipfile-module for info in archive.infolist(): out_path = os.path.join(target_directory, info.filename) @@ -87,6 +88,7 @@ def unstream_dir(stream, length, target_directory): archive.extract(info.filename, path=target_directory) # Fancy logic to preserve modification times + # AWX uses modification times to determine if new facts were written for a host # https://stackoverflow.com/questions/9813243/extract-files-from-zip-file-and-retain-mod-date date_time = time.mktime(info.date_time + (0, 0, -1)) os.utime(out_path, times=(date_time, date_time)) From e40bfbb05de7691c6034b44f86e595422c8649d1 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 26 Jan 2022 15:00:19 -0500 Subject: [PATCH 3/3] Fix test bug and explain the weirdness of 2 seconds --- test/unit/utils/test_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/unit/utils/test_utils.py b/test/unit/utils/test_utils.py index 740cfa851..860a099f7 100644 --- a/test/unit/utils/test_utils.py +++ b/test/unit/utils/test_utils.py @@ -153,8 +153,10 @@ def test_transmit_modtimes(tmp_path): source_dir = tmp_path / 'source' source_dir.mkdir() + # python ZipFile uses an old standard that stores seconds in 2 second increments + # https://stackoverflow.com/questions/64048499/zipfile-lib-weird-behaviour-with-seconds-in-modified-time (source_dir / 'b.txt').touch() - time.sleep(2.0) # flaky for anything less because of integer math + time.sleep(2.0) # flaky for anything less (source_dir / 'a.txt').touch() very_old_file = source_dir / 'very_old.txt' @@ -182,12 +184,13 @@ def test_transmit_modtimes(tmp_path): mod_delta = os.path.getmtime(dest_dir / 'a.txt') - os.path.getmtime(dest_dir / 'b.txt') assert mod_delta >= 1.0 - # Assure modification times are same as original - for filename in ('a.txt', 'b.txt'): - assert os.path.getmtime(dest_dir / filename) == os.path.getmtime(dest_dir / filename) + # Assure modification times are same as original (to the rounded second) + for filename in ('a.txt', 'b.txt', 'very_old.txt'): + difference = abs(os.path.getmtime(dest_dir / filename) - os.path.getmtime(source_dir / filename)) + assert difference < 2.0 # Assure the very old timestamp is preserved - old_delta = os.path.getmtime(dest_dir / 'a.txt') - os.path.getmtime(dest_dir / 'very_old.txt') + old_delta = os.path.getmtime(dest_dir / 'a.txt') - os.path.getmtime(source_dir / 'very_old.txt') assert old_delta >= datetime.timedelta(days=1).total_seconds() - 2.