From ee0d9e3630232530dcc26b614af809726463060b Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:43:14 -0400 Subject: [PATCH 01/13] Add file_utils.touch(), prep for file ops atime and mtime metadata support Add a restricted analog to the ``touch`` command. Supports mutating atime and mtime, however only allows modification of one time field at a time. A single unambiguous datetime val is the time source since any ``touch`` operation with specified atime/mtime requires the month, day, and time at a minimum. Is BSD aware in that datetimes are always converted to UTC rather than relying on the richer timezone support in GNU ``touch``. The single time field setting approach is less flexible per invocation than native ``touch``, but also far simpler and nicely fits with file operations and their way of working on a single attribute at a time. --- pyinfra/operations/util/files.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pyinfra/operations/util/files.py b/pyinfra/operations/util/files.py index 0dd48506f..8b51d52fa 100644 --- a/pyinfra/operations/util/files.py +++ b/pyinfra/operations/util/files.py @@ -1,7 +1,7 @@ from __future__ import annotations import re -from datetime import datetime +from datetime import datetime, timezone from pyinfra.api import QuoteString, StringCommand @@ -115,6 +115,36 @@ def chown( return StringCommand(" ".join(args), user_group, QuoteString(target)) +# like the touch command, but only supports setting one field at a time, and expects any +# reference times to have been read from the reference file metadata and turned into +# aware datetimes +def touch( + target: str, + atime_or_mtime: str, + timesrc: datetime, + dereference=True, +) -> StringCommand: + args = ["touch"] + + if atime_or_mtime == "atime": + args.append("-a") + elif atime_or_mtime == "mtime": + args.append("-m") + else: + ValueError("Bad argument `atime_or_mtime`: {0}".format(atime_or_mtime)) + + if not dereference: + args.append("-h") + + # don't reinvent the wheel; use isoformat() + timestr = timesrc.astimezone(timezone.utc).isoformat() + # but replace the ISO format TZ offset with "Z" for BSD + timestr = timestr.replace("+00:00", "Z") + args.extend(["-d", timestr]) + + return StringCommand(" ".join(args), QuoteString(target)) + + def adjust_regex(line: str, escape_regex_characters: bool) -> str: """ Ensure the regex starts with '^' and ends with '$' and escape regex characters if requested From 8153492ed815a59ae43d969765c278779fb46d49 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 7 Jun 2025 21:54:03 -0400 Subject: [PATCH 02/13] Add support for atime and mtime in files.put() Time metadate updates are not coalesced, as ``touch`` can do with a single date and time arg, or via reference files. However, this actually allows more flexibility than native ''touch`` by allowing both atime and mtime values to be specified independently. Two support functions are added as well. --- pyinfra/operations/files.py | 115 +++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index edd9d2bae..3b640fe0f 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -8,7 +8,7 @@ import posixpath import sys import traceback -from datetime import timedelta +from datetime import datetime, timedelta, timezone from fnmatch import fnmatch from io import StringIO from pathlib import Path @@ -778,6 +778,56 @@ def get( yield FileDownloadCommand(src, dest, remote_temp_filename=host.get_temp_filename(dest)) +def _canonicalize_timespec(field, local_file, timespec): + assert field == "atime" or field == "mtime" + if isinstance(timespec, datetime): + if not timespec.tzinfo: + # specify remote host timezone + timespec_with_tz = timespec.replace(tzinfo=host.get_fact(Date).tzinfo) + return timespec_with_tz + else: + return timespec + elif isinstance(timespec, bool) and timespec: + lf_stat = os.stat(local_file) + if field == "atime": + return datetime.fromtimestamp(lf_stat.st_atime, tz=timezone.utc) + else: + return datetime.fromtimestamp(lf_stat.st_mtime, tz=timezone.utc) + else: + try: + isodatetime = datetime.fromisoformat(timespec) + if not isodatetime.tzinfo: + return isodatetime.replace(tzinfo=host.get_fact(Date).tzinfo) + else: + return isodatetime + except ValueError: + try: + timestamp = float(timespec) + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + except ValueError: + # verify there is a remote file matching path in timesrc + ref_file = host.get_fact(File, path=timespec) + if ref_file: + if field == "atime": + assert ref_file["atime"] is not None + return ref_file["atime"].replace(tzinfo=timezone.utc) + else: + assert ref_file["mtime"] is not None + return ref_file["mtime"].replace(tzinfo=timezone.utc) + else: + ValueError("Bad argument for `timesspec`: {0}".format(timespec)) + + +# returns True for a visible difference in the second field between the datetime values +# in the ref's TZ +def _times_differ_in_s(ref, cand): + assert ref.tzinfo and cand.tzinfo + cand_in_ref_tz = cand.astimezone(ref.tzinfo) + return (abs((cand_in_ref_tz - ref).total_seconds()) >= 1.0) or ( + ref.second != cand_in_ref_tz.second + ) + + @operation() def put( src: str | IO[Any], @@ -789,6 +839,8 @@ def put( create_remote_dir=True, force=False, assume_exists=False, + atime: datetime | float | int | str | bool | None = None, + mtime: datetime | float | int | str | bool | None = None, ): """ Upload a local file, or file-like object, to the remote system. @@ -802,6 +854,9 @@ def put( + create_remote_dir: create the remote directory if it doesn't exist + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists + + atime: value of atime the file should have, use ``True`` to match the local file + + mtime: value of atime the file should have, use ``True`` to match the local file + + timesrc: the source of the time value if atime or mtime are ``True`` ``dest``: If this is a directory that already exists on the remote side, the local @@ -818,7 +873,21 @@ def put( user & group as passed to ``files.put``. The mode will *not* be copied over, if this is required call ``files.directory`` separately. - Note: + ``atime`` and ``mtime``: + When set to values other than ``False`` or ``None``, the respective metadata + fields on the remote file will updated accordingly. Timestamp values are + considered equivalent if the difference is less than one second and they have + the identical number in the seconds field. If set to ``True`` the local + file is the source of the value. Otherwise, these values can be provided as + ``datetime`` objects, POSIX timestamps, or strings that can be parsed into + either of these date and time specifications. They can also be reference file + paths on the remote host, as with the ``-r`` argument to ``touch``. If a + ``datetime`` argument has no ``tzinfo`` value (i.e., it is naive), it is + assumed to be in the remote host's local timezone. There is no shortcut for + setting both ``atime` and ``mtime`` values with a single time specification, + unlike the native ``touch`` command. + + Notes: This operation is not suitable for large files as it may involve copying the file before uploading it. @@ -827,6 +896,12 @@ def put( behave as if the remote file does not match the specified permissions and requires a change. + If the ``atime`` argument is set for a given file, unless the remote + filesystem is mounted ``noatime`` or ``relatime``, multiple runs of this + operation will trigger the change detection for that file, since the act of + reading and checksumming the file will cause the host OS to update the file's + ``atime``. + **Examples:** .. code:: python @@ -902,12 +977,20 @@ def put( if mode: yield file_utils.chmod(dest, mode) - # File exists, check sum and check user/group/mode if supplied + # do mtime before atime to ensure atime setting isn't undone by mtime setting + if mtime: + yield file_utils.touch(dest, "mtime", _canonicalize_timespec("mtime", src, mtime)) + + if atime: + yield file_utils.touch(dest, "atime", _canonicalize_timespec("atime", src, atime)) + + # File exists, check sum and check user/group/mode/atime/mtime if supplied else: remote_sum = host.get_fact(Sha1File, path=dest) # Check sha1sum, upload if needed if local_sum != remote_sum: + yield FileUploadCommand( local_file, dest, @@ -920,6 +1003,12 @@ def put( if mode: yield file_utils.chmod(dest, mode) + if mtime: + yield file_utils.touch(dest, "mtime", _canonicalize_timespec("mtime", src, mtime)) + + if atime: + yield file_utils.touch(dest, "atime", _canonicalize_timespec("atime", src, atime)) + else: changed = False @@ -933,6 +1022,26 @@ def put( yield file_utils.chown(dest, user, group) changed = True + # Check mtime + if mtime: + canonical_mtime = _canonicalize_timespec("mtime", src, mtime) + assert remote_file["mtime"] is not None + if _times_differ_in_s( + canonical_mtime, remote_file["mtime"].replace(tzinfo=timezone.utc) + ): + yield file_utils.touch(dest, "mtime", canonical_mtime) + changed = True + + # Check atime + if atime: + canonical_atime = _canonicalize_timespec("atime", src, atime) + assert remote_file["atime"] is not None + if _times_differ_in_s( + canonical_atime, remote_file["atime"].replace(tzinfo=timezone.utc) + ): + yield file_utils.touch(dest, "atime", canonical_atime) + changed = True + if not changed: host.noop("file {0} is already uploaded".format(dest)) From d7bc21b8b90d19c741ebb60e5c1e50655eb65725 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Tue, 10 Jun 2025 01:53:12 -0400 Subject: [PATCH 03/13] Fix the stat regex in facts/files.py to allow negative timestamps Negative timestamps are valid and permissible. Ensure pyinfra.facts.files.stat doesn't fail to match stat command output when such values occur. --- pyinfra/facts/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinfra/facts/files.py b/pyinfra/facts/files.py index fa925e24a..53a7058e7 100644 --- a/pyinfra/facts/files.py +++ b/pyinfra/facts/files.py @@ -27,7 +27,7 @@ STAT_REGEX = ( r"user=(.*) group=(.*) mode=(.*) " - r"atime=([0-9]*) mtime=([0-9]*) ctime=([0-9]*) " + r"atime=(-?[0-9]*) mtime=(-?[0-9]*) ctime=(-?[0-9]*) " r"size=([0-9]*) (.*)" ) From 308f118414ac487f178a932b238d7654074c2cdf Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Tue, 10 Jun 2025 01:38:17 -0400 Subject: [PATCH 04/13] Rewrite mock os.stat in tests/util.py and apply to pyinfra.operations.files In preparation for tests of mtime and ctime support, make the mock os.stat actually use any stat fields provided in the test json, with non-zero defaults for all fields. Doesn't break any existing test, but lays foundation for any future ops or facts which may use this info. Doesn't currently support setting the st_{a, m}time_ns high-resolution time fields. --- tests/util.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/tests/util.py b/tests/util.py index 204ba6bcb..939e2f3a8 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,6 +1,8 @@ +import copy import json import os -from datetime import datetime +import re +from datetime import datetime, timezone from inspect import getcallargs, getfullargspec from os import path from pathlib import Path @@ -308,6 +310,7 @@ def __enter__(self): patch("pyinfra.operations.files.os.path.isfile", self.isfile), patch("pyinfra.operations.files.os.path.isdir", self.isdir), patch("pyinfra.operations.files.os.walk", self.walk), + patch("pyinfra.operations.files.os.stat", self.stat), patch("pyinfra.operations.files.os.makedirs", lambda path: True), patch("pyinfra.api.util.stat", self.stat), # Builtin patches @@ -342,14 +345,98 @@ def isdir(self, dirname, *args): return normalized_path in self._directories def stat(self, pathname): + try: + fileinfo = copy.deepcopy(self._files_data[pathname]) + if not fileinfo: + fileinfo = dict() + except KeyError: + fileinfo = dict() + if self.isfile(pathname): - mode_int = 33188 # 644 file + default_mode = 33188 # 644 file elif self.isdir(pathname): - mode_int = 16877 # 755 directory + default_mode = 16877 # 755 directory else: raise IOError("No such file or directory: {0}".format(pathname)) - return os.stat_result((mode_int, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + default_timeval = datetime.fromisoformat("2008-08-09T13:21:44Z").timestamp() + defaults = dict( + mode=default_mode, ino=64321, dev=64556, nlink=1, uid=1001, gid=1001, size=10240 + ) + + if "mode" in fileinfo.keys(): + if isinstance(fileinfo["mode"], str): + perms = int(fileinfo["mode"], 8) + else: + # this assumes the mode was provided as an integer whose digits are really octal + perms = int(str(fileinfo["mode"]), 8) + + if self.isfile(pathname): + fileinfo["mode"] = 0o100000 + perms + else: + fileinfo["mode"] = 0o40000 + perms + else: + fileinfo["mode"] = defaults["mode"] + + for field in ["dev", "nlink", "uid", "gid", "size"]: + if field in fileinfo.keys(): + if isinstance(fileinfo[field], str): + fileinfo[field] = int(fileinfo[field]) + else: + fileinfo[field] = defaults[field] + + # support both "ino" and "inode" as keys for st_ino + if "ino" in fileinfo.keys(): + if isinstance(fileinfo["ino"], str): + fileinfo["ino"] = int(fileinfo["ino"]) + elif "inode" in fileinfo.keys(): + if isinstance(fileinfo["inode"], str): + fileinfo["ino"] = int(fileinfo["inode"]) + else: + fileinfo["ino"] = fileinfo["inode"] + else: + fileinfo["ino"] = defaults["ino"] + + for timefield in ["atime", "mtime", "ctime"]: + if timefield in fileinfo.keys(): + if not isinstance(fileinfo[timefield], (int, float, str)): + raise TypeError("Parameter {0} must have type float, int or str", timefield) + + if isinstance(fileinfo[timefield], str): + if fileinfo[timefield].startswith("datetime:"): + timestr = fileinfo[timefield].removeprefix("datetime:") + dt = datetime.fromisoformat(timestr.strip()) + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + fileinfo[timefield] = dt.timestamp() + elif re.match("^-?[0-9]+$", fileinfo[timefield].strip()): + fileinfo[timefield] = int(fileinfo[timefield].strip()) + elif re.match("^-?[0-9]+(\\.[0-9]*)?$", fileinfo[timefield].strip()): + fileinfo[timefield] = float(fileinfo[timefield].strip()) + else: + raise ValueError( + "Invalid argument: {0} for {1}", fileinfo[timefield], timefield + ) + else: + fileinfo[timefield] = default_timeval + + return os.stat_result( + tuple( + fileinfo[field] + for field in [ + "mode", + "ino", + "dev", + "nlink", + "uid", + "gid", + "size", + "atime", + "mtime", + "ctime", + ] + ) + ) def walk(self, dirname, topdown=True, onerror=None, followlinks=False): if not self.isdir(dirname): From f90ec08e560b28e3fbcf9593b75c269f78a8f0c1 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Tue, 10 Jun 2025 02:12:51 -0400 Subject: [PATCH 05/13] Change test/utils.py parse_value() to use datetime.fromisoformat Change the datetime Python value to be generated from the fromisoformat method rather than strptime. This yields identical results as before when no UTC offset information is provided, but respects it if it does. It is also consistent with the os.stat parsing. --- tests/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/util.py b/tests/util.py index 939e2f3a8..a19c3b7c3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -67,7 +67,7 @@ def parse_value(value): if isinstance(value, str): if value.startswith("datetime:"): - return datetime.strptime(value[9:], "%Y-%m-%dT%H:%M:%S") + return datetime.fromisoformat(value[9:]) if value.startswith("path:"): return Path(value[5:]) return value From 95345cc2cd6179f11666631472e5fe3b4b1b827b Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Tue, 10 Jun 2025 21:00:39 -0400 Subject: [PATCH 06/13] Add tests for ops/files.put atime and mtime support Add several testcases to cover multiple aspects of functionality: - Change detection, incl. differences <1s that are visible in the seconds vals - Separate actions on mtime and atime - Local files as the time references - Remote files as the time references - Use of the server's timezone when a datetime arg has no tzinfo --- .../files.put/atime_mtime_sub_second.json | 31 ++++++++++++++++ .../change_atime_eq_mtime_ref_remote.json | 37 +++++++++++++++++++ .../files.put/change_mtime_arg_no_tz.json | 37 +++++++++++++++++++ .../change_mtime_atime_ref_local file.json | 37 +++++++++++++++++++ .../files.put/no_change_atime_mtime.json | 35 ++++++++++++++++++ 5 files changed, 177 insertions(+) create mode 100644 tests/operations/files.put/atime_mtime_sub_second.json create mode 100644 tests/operations/files.put/change_atime_eq_mtime_ref_remote.json create mode 100644 tests/operations/files.put/change_mtime_arg_no_tz.json create mode 100644 tests/operations/files.put/change_mtime_atime_ref_local file.json create mode 100644 tests/operations/files.put/no_change_atime_mtime.json diff --git a/tests/operations/files.put/atime_mtime_sub_second.json b/tests/operations/files.put/atime_mtime_sub_second.json new file mode 100644 index 000000000..15f83db0c --- /dev/null +++ b/tests/operations/files.put/atime_mtime_sub_second.json @@ -0,0 +1,31 @@ +{ + "args": ["somefile.txt", "/home/somefile.txt"], + "kwargs": { + "atime": "datetime:2002-09-15T10:11:12Z", + "mtime": "datetime:2002-09-15T10:11:11.888888Z", + }, + "local_files": { + "files": { + "somefile.txt": null + }, + "dirs": {} + }, + "facts": { + "files.File": { + "path=/home/somefile.txt": { + "mode": 640, + "atime": "datetime:2002-09-15T10:11:12", + "mtime": "datetime:2002-09-15T10:11:12" + }, + }, + "files.Directory": { + "path=/home": true, + }, + "files.Sha1File": { + "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" + } + }, + "commands": [ + "touch -m -d 2002-09-15T10:11:11.888888Z /home/somefile.txt", + ] +} diff --git a/tests/operations/files.put/change_atime_eq_mtime_ref_remote.json b/tests/operations/files.put/change_atime_eq_mtime_ref_remote.json new file mode 100644 index 000000000..c373274da --- /dev/null +++ b/tests/operations/files.put/change_atime_eq_mtime_ref_remote.json @@ -0,0 +1,37 @@ +{ + "args": ["somefile.txt", "/home/somefile.txt"], + "kwargs": { + "atime": "/var/timeref.blob", + "mtime": "/var/timeref.blob", + }, + "local_files": { + "files": { + "somefile.txt": null + }, + "dirs": {} + }, + "facts": { + "files.File": { + "path=/home/somefile.txt": { + "mode": 500, + "atime": "datetime:2020-07-15T09:19:27", + "mtime": "datetime:2002-09-15T10:11:12" + }, + "path=/var/timeref.blob": { + "mode": 644, + "atime": "datetime:1991-04-15T18:18:27", + "mtime": "datetime:2002-09-15T10:11:12" + }, + }, + "files.Directory": { + "path=/home": true, + "path=/var": true + }, + "files.Sha1File": { + "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" + } + }, + "commands": [ + "touch -a -d 1991-04-15T18:18:27Z /home/somefile.txt", + ] +} diff --git a/tests/operations/files.put/change_mtime_arg_no_tz.json b/tests/operations/files.put/change_mtime_arg_no_tz.json new file mode 100644 index 000000000..7a5972f3d --- /dev/null +++ b/tests/operations/files.put/change_mtime_arg_no_tz.json @@ -0,0 +1,37 @@ +{ + "args": ["somefile.txt", "/home/somefile.txt"], + "kwargs": { + "atime": false, + "mtime": "2022-11-17T20:45:00", + }, + "local_files": { + "files": { + "somefile.txt": { + "mode": 644, + "ctime": "datetime:1997-04-21T18:06:55.982Z", + "atime": "datetime:2020-06-20T22:09:17", + "mtime": "datetime:2000-05-01T00:01:00Z" + }, + }, + "dirs": {} + }, + "facts": { + "files.File": { + "path=/home/somefile.txt": { + "mode": 500, + "atime": "datetime:2020-07-15T09:19:27", + "mtime": "datetime:2020-07-15T09:19:27" + } + }, + "files.Directory": { + "path=/home": true + }, + "files.Sha1File": { + "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" + }, + "server.Date": "datetime:2017-03-03T10:07:37-06:00" + }, + "commands": [ + "touch -m -d 2022-11-18T02:45:00Z /home/somefile.txt", + ] +} diff --git a/tests/operations/files.put/change_mtime_atime_ref_local file.json b/tests/operations/files.put/change_mtime_atime_ref_local file.json new file mode 100644 index 000000000..114a29ffe --- /dev/null +++ b/tests/operations/files.put/change_mtime_atime_ref_local file.json @@ -0,0 +1,37 @@ +{ + "args": ["somefile.txt", "/home/somefile.txt"], + "kwargs": { + "atime": true, + "mtime": true, + }, + "local_files": { + "files": { + "somefile.txt": { + "mode": 644, + "ctime": "datetime:1997-04-21T18:06:55.982Z", + "atime": "datetime:2020-06-20T22:09:17", + "mtime": "datetime:2000-05-01T00:01:00Z" + }, + }, + "dirs": {} + }, + "facts": { + "files.File": { + "path=/home/somefile.txt": { + "mode": 500, + "atime": "datetime:2020-07-15T09:19:27", + "mtime": "datetime:2020-07-15T09:19:27" + } + }, + "files.Directory": { + "path=/home": true + }, + "files.Sha1File": { + "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" + } + }, + "commands": [ + "touch -m -d 2000-05-01T00:01:00Z /home/somefile.txt", + "touch -a -d 2020-06-20T22:09:17Z /home/somefile.txt", + ] +} diff --git a/tests/operations/files.put/no_change_atime_mtime.json b/tests/operations/files.put/no_change_atime_mtime.json new file mode 100644 index 000000000..4e0eae036 --- /dev/null +++ b/tests/operations/files.put/no_change_atime_mtime.json @@ -0,0 +1,35 @@ +{ + "args": ["somefile.txt", "/home/somefile.txt"], + "kwargs": { + "atime": "datetime:2020-02-20T20:20:20Z", + "mtime": true, + }, + "local_files": { + "files": { + "somefile.txt": { + "mode": 644, + "ctime": "datetime:1997-04-21T18:06:55.982Z", + "atime": "datetime:2020-06-20T22:09:17", + "mtime": "datetime:2000-05-01T00:01:00Z" + }, + }, + "dirs": {} + }, + "facts": { + "files.File": { + "path=/home/somefile.txt": { + "mode": 640, + "atime": "datetime:2020-02-20T20:20:20", + "mtime": "datetime:2000-05-01T00:01:00" + }, + }, + "files.Directory": { + "path=/home": true, + }, + "files.Sha1File": { + "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" + } + }, + "commands": [], + "noop_description": "file /home/somefile.txt is already uploaded" +} From 575b9c24ec7b98efad00e34b306cc8b910c147b4 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 12 Jul 2025 22:17:48 -0400 Subject: [PATCH 07/13] Remove extraneous dangling argument docstring leftover from prior attempt. --- pyinfra/operations/files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index 3b640fe0f..184ff33a9 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -856,7 +856,6 @@ def put( + assume_exists: whether to assume the local file exists + atime: value of atime the file should have, use ``True`` to match the local file + mtime: value of atime the file should have, use ``True`` to match the local file - + timesrc: the source of the time value if atime or mtime are ``True`` ``dest``: If this is a directory that already exists on the remote side, the local From f61fb6e57991c7bea3bfdda832556a8b96e2aaf9 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 12 Jul 2025 23:20:02 -0400 Subject: [PATCH 08/13] Use enum as flag for atime and mtime in internal functions. In operations/util/files.py, define an enum to indicate whether the metadata time field being manipulated by touch() is atime or mtime.` This feeds into operations/files.py and the _canonicalize_timespec() support function. --- pyinfra/operations/files.py | 46 +++++++++++++++++++++++--------- pyinfra/operations/util/files.py | 14 ++++++---- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index 184ff33a9..b79d04fbc 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -56,7 +56,14 @@ from pyinfra.facts.server import Date, Which from .util import files as file_utils -from .util.files import adjust_regex, ensure_mode_int, get_timestamp, sed_replace, unix_path_join +from .util.files import ( + MetadataTimeField, + adjust_regex, + ensure_mode_int, + get_timestamp, + sed_replace, + unix_path_join, +) @operation() @@ -778,8 +785,7 @@ def get( yield FileDownloadCommand(src, dest, remote_temp_filename=host.get_temp_filename(dest)) -def _canonicalize_timespec(field, local_file, timespec): - assert field == "atime" or field == "mtime" +def _canonicalize_timespec(field: MetadataTimeField, local_file, timespec): if isinstance(timespec, datetime): if not timespec.tzinfo: # specify remote host timezone @@ -808,7 +814,7 @@ def _canonicalize_timespec(field, local_file, timespec): # verify there is a remote file matching path in timesrc ref_file = host.get_fact(File, path=timespec) if ref_file: - if field == "atime": + if field is MetadataTimeField.ATIME: assert ref_file["atime"] is not None return ref_file["atime"].replace(tzinfo=timezone.utc) else: @@ -978,10 +984,18 @@ def put( # do mtime before atime to ensure atime setting isn't undone by mtime setting if mtime: - yield file_utils.touch(dest, "mtime", _canonicalize_timespec("mtime", src, mtime)) + yield file_utils.touch( + dest, + MetadataTimeField.MTIME, + _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime), + ) if atime: - yield file_utils.touch(dest, "atime", _canonicalize_timespec("atime", src, atime)) + yield file_utils.touch( + dest, + MetadataTimeField.ATIME, + _canonicalize_timespec(MetadataTimeField.ATIME, src, atime), + ) # File exists, check sum and check user/group/mode/atime/mtime if supplied else: @@ -1003,10 +1017,18 @@ def put( yield file_utils.chmod(dest, mode) if mtime: - yield file_utils.touch(dest, "mtime", _canonicalize_timespec("mtime", src, mtime)) + yield file_utils.touch( + dest, + MetadataTimeField.MTIME, + _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime), + ) if atime: - yield file_utils.touch(dest, "atime", _canonicalize_timespec("atime", src, atime)) + yield file_utils.touch( + dest, + MetadataTimeField.ATIME, + _canonicalize_timespec(MetadataTimeField.ATIME, src, atime), + ) else: changed = False @@ -1023,22 +1045,22 @@ def put( # Check mtime if mtime: - canonical_mtime = _canonicalize_timespec("mtime", src, mtime) + canonical_mtime = _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime) assert remote_file["mtime"] is not None if _times_differ_in_s( canonical_mtime, remote_file["mtime"].replace(tzinfo=timezone.utc) ): - yield file_utils.touch(dest, "mtime", canonical_mtime) + yield file_utils.touch(dest, MetadataTimeField.MTIME, canonical_mtime) changed = True # Check atime if atime: - canonical_atime = _canonicalize_timespec("atime", src, atime) + canonical_atime = _canonicalize_timespec(MetadataTimeField.ATIME, src, atime) assert remote_file["atime"] is not None if _times_differ_in_s( canonical_atime, remote_file["atime"].replace(tzinfo=timezone.utc) ): - yield file_utils.touch(dest, "atime", canonical_atime) + yield file_utils.touch(dest, MetadataTimeField.ATIME, canonical_atime) changed = True if not changed: diff --git a/pyinfra/operations/util/files.py b/pyinfra/operations/util/files.py index 8b51d52fa..6e87f1919 100644 --- a/pyinfra/operations/util/files.py +++ b/pyinfra/operations/util/files.py @@ -2,10 +2,16 @@ import re from datetime import datetime, timezone +from enum import Enum from pyinfra.api import QuoteString, StringCommand +class MetadataTimeField(Enum): + ATIME = "atime" + MTIME = "mtime" + + def unix_path_join(*parts) -> str: part_list = list(parts) part_list[0:-1] = [part.rstrip("/") for part in part_list[0:-1]] @@ -120,18 +126,16 @@ def chown( # aware datetimes def touch( target: str, - atime_or_mtime: str, + timefield: MetadataTimeField, timesrc: datetime, dereference=True, ) -> StringCommand: args = ["touch"] - if atime_or_mtime == "atime": + if timefield is MetadataTimeField.ATIME: args.append("-a") - elif atime_or_mtime == "mtime": - args.append("-m") else: - ValueError("Bad argument `atime_or_mtime`: {0}".format(atime_or_mtime)) + args.append("-m") if not dereference: args.append("-h") From f5ae4efdb55306efee2d2a07609774ebab1a35a6 Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 12 Jul 2025 23:33:25 -0400 Subject: [PATCH 09/13] Fix a copypasta miss in the docstring for the mtime arg. --- pyinfra/operations/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index b79d04fbc..cf3ff0489 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -861,7 +861,7 @@ def put( + force: always upload the file, even if the remote copy matches + assume_exists: whether to assume the local file exists + atime: value of atime the file should have, use ``True`` to match the local file - + mtime: value of atime the file should have, use ``True`` to match the local file + + mtime: value of mtime the file should have, use ``True`` to match the local file ``dest``: If this is a directory that already exists on the remote side, the local From b57a3ff9c51c68d015e2c24d0e2650efb1fa147a Mon Sep 17 00:00:00 2001 From: Vijay Ram <213466426+vram0gh2@users.noreply.github.com> Date: Sat, 12 Jul 2025 23:34:32 -0400 Subject: [PATCH 10/13] Simplify a conditional block in operations/files.py:_canonicalize_timespec() --- pyinfra/operations/files.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index cf3ff0489..3a8b38213 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -794,11 +794,12 @@ def _canonicalize_timespec(field: MetadataTimeField, local_file, timespec): else: return timespec elif isinstance(timespec, bool) and timespec: - lf_stat = os.stat(local_file) - if field == "atime": - return datetime.fromtimestamp(lf_stat.st_atime, tz=timezone.utc) - else: - return datetime.fromtimestamp(lf_stat.st_mtime, tz=timezone.utc) + lf_ts = ( + os.stat(local_file).st_atime + if field is MetadataTimeField.ATIME + else os.stat(local_file).st_mtime + ) + return datetime.fromtimestamp(lf_ts, tz=timezone.utc) else: try: isodatetime = datetime.fromisoformat(timespec) From 1c7bbff5a6549942f88ce1a806c8d662abfef2e2 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 23 Jul 2025 11:13:09 +0100 Subject: [PATCH 11/13] Fix python <3.11 isoformat parsing --- tests/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/util.py b/tests/util.py index a378d30d7..100c5e169 100644 --- a/tests/util.py +++ b/tests/util.py @@ -360,7 +360,7 @@ def stat(self, pathname): else: raise IOError("No such file or directory: {0}".format(pathname)) - default_timeval = datetime.fromisoformat("2008-08-09T13:21:44Z").timestamp() + default_timeval = datetime.fromisoformat("2008-08-09T13:21:44").timestamp() defaults = dict( mode=default_mode, ino=64321, dev=64556, nlink=1, uid=1001, gid=1001, size=10240 ) From 8e6336e06542a33f61a164732ec838c9a279b2a2 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 23 Jul 2025 19:26:51 +0100 Subject: [PATCH 12/13] Use dates Python 3.10 can handle in tests --- .../files.put/atime_mtime_sub_second.json | 10 +++++----- .../change_mtime_atime_ref_local file.json | 12 ++++++------ .../files.put/no_change_atime_mtime.json | 14 +++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/operations/files.put/atime_mtime_sub_second.json b/tests/operations/files.put/atime_mtime_sub_second.json index 15f83db0c..f9f1c8e96 100644 --- a/tests/operations/files.put/atime_mtime_sub_second.json +++ b/tests/operations/files.put/atime_mtime_sub_second.json @@ -1,8 +1,8 @@ { "args": ["somefile.txt", "/home/somefile.txt"], "kwargs": { - "atime": "datetime:2002-09-15T10:11:12Z", - "mtime": "datetime:2002-09-15T10:11:11.888888Z", + "atime": "datetime:2002-09-15T10:11:12+00:00", + "mtime": "datetime:2002-09-15T10:11:11.888888+00:00" }, "local_files": { "files": { @@ -16,16 +16,16 @@ "mode": 640, "atime": "datetime:2002-09-15T10:11:12", "mtime": "datetime:2002-09-15T10:11:12" - }, + } }, "files.Directory": { - "path=/home": true, + "path=/home": true }, "files.Sha1File": { "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" } }, "commands": [ - "touch -m -d 2002-09-15T10:11:11.888888Z /home/somefile.txt", + "touch -m -d 2002-09-15T10:11:11.888888Z /home/somefile.txt" ] } diff --git a/tests/operations/files.put/change_mtime_atime_ref_local file.json b/tests/operations/files.put/change_mtime_atime_ref_local file.json index 114a29ffe..c4bced86c 100644 --- a/tests/operations/files.put/change_mtime_atime_ref_local file.json +++ b/tests/operations/files.put/change_mtime_atime_ref_local file.json @@ -2,16 +2,16 @@ "args": ["somefile.txt", "/home/somefile.txt"], "kwargs": { "atime": true, - "mtime": true, + "mtime": true }, "local_files": { "files": { "somefile.txt": { "mode": 644, - "ctime": "datetime:1997-04-21T18:06:55.982Z", - "atime": "datetime:2020-06-20T22:09:17", - "mtime": "datetime:2000-05-01T00:01:00Z" - }, + "ctime": "datetime:1997-04-21T18:06:55.982+00:00", + "atime": "datetime:2020-06-20T22:09:17+00:00", + "mtime": "datetime:2000-05-01T00:01:00+00:00" + } }, "dirs": {} }, @@ -32,6 +32,6 @@ }, "commands": [ "touch -m -d 2000-05-01T00:01:00Z /home/somefile.txt", - "touch -a -d 2020-06-20T22:09:17Z /home/somefile.txt", + "touch -a -d 2020-06-20T22:09:17Z /home/somefile.txt" ] } diff --git a/tests/operations/files.put/no_change_atime_mtime.json b/tests/operations/files.put/no_change_atime_mtime.json index 4e0eae036..2225cac5b 100644 --- a/tests/operations/files.put/no_change_atime_mtime.json +++ b/tests/operations/files.put/no_change_atime_mtime.json @@ -1,17 +1,17 @@ { "args": ["somefile.txt", "/home/somefile.txt"], "kwargs": { - "atime": "datetime:2020-02-20T20:20:20Z", - "mtime": true, + "atime": "datetime:2020-02-20T20:20:20+00:00", + "mtime": true }, "local_files": { "files": { "somefile.txt": { "mode": 644, - "ctime": "datetime:1997-04-21T18:06:55.982Z", + "ctime": "datetime:1997-04-21T18:06:55.982+00:00", "atime": "datetime:2020-06-20T22:09:17", - "mtime": "datetime:2000-05-01T00:01:00Z" - }, + "mtime": "datetime:2000-05-01T00:01:00+00:00" + } }, "dirs": {} }, @@ -21,10 +21,10 @@ "mode": 640, "atime": "datetime:2020-02-20T20:20:20", "mtime": "datetime:2000-05-01T00:01:00" - }, + } }, "files.Directory": { - "path=/home": true, + "path=/home": true }, "files.Sha1File": { "path=/home/somefile.txt": "ac2cd59a622114712b5b21081763c54bf0caacb8" From e4a78cb2f3b31d87d094c7d19dd4016f87ced119 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 23 Jul 2025 20:26:45 +0100 Subject: [PATCH 13/13] Attempt to fix windows tests Unfortunately I have no windows machine readily available, so this is guesswork. --- .../files.put/change_mtime_atime_ref_local file.json | 7 +++++++ .../operations/files.put/no_change_atime_mtime.json | 12 ++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/operations/files.put/change_mtime_atime_ref_local file.json b/tests/operations/files.put/change_mtime_atime_ref_local file.json index c4bced86c..e9e50f406 100644 --- a/tests/operations/files.put/change_mtime_atime_ref_local file.json +++ b/tests/operations/files.put/change_mtime_atime_ref_local file.json @@ -11,6 +11,13 @@ "ctime": "datetime:1997-04-21T18:06:55.982+00:00", "atime": "datetime:2020-06-20T22:09:17+00:00", "mtime": "datetime:2000-05-01T00:01:00+00:00" + }, + "/somefile.txt": { + "TODO": "this (dupe /somefile.txt fake file) is required for windows CI to pass, fixme", + "mode": 644, + "ctime": "datetime:1997-04-21T18:06:55.982+00:00", + "atime": "datetime:2020-06-20T22:09:17+00:00", + "mtime": "datetime:2000-05-01T00:01:00+00:00" } }, "dirs": {} diff --git a/tests/operations/files.put/no_change_atime_mtime.json b/tests/operations/files.put/no_change_atime_mtime.json index 2225cac5b..25706f754 100644 --- a/tests/operations/files.put/no_change_atime_mtime.json +++ b/tests/operations/files.put/no_change_atime_mtime.json @@ -1,7 +1,7 @@ { "args": ["somefile.txt", "/home/somefile.txt"], "kwargs": { - "atime": "datetime:2020-02-20T20:20:20+00:00", + "atime": "datetime:2020-02-20T20:20:20", "mtime": true }, "local_files": { @@ -9,13 +9,21 @@ "somefile.txt": { "mode": 644, "ctime": "datetime:1997-04-21T18:06:55.982+00:00", - "atime": "datetime:2020-06-20T22:09:17", + "atime": "datetime:2020-06-20T22:09:17+00:00", + "mtime": "datetime:2000-05-01T00:01:00+00:00" + }, + "/somefile.txt": { + "TODO": "this (dupe /somefile.txt fake file) is required for windows CI to pass, fixme", + "mode": 644, + "ctime": "datetime:1997-04-21T18:06:55.982+00:00", + "atime": "datetime:2020-06-20T22:09:17+00:00", "mtime": "datetime:2000-05-01T00:01:00+00:00" } }, "dirs": {} }, "facts": { + "server.Date": "datetime:2015-01-01T00:00:00+00:00", "files.File": { "path=/home/somefile.txt": { "mode": 640,