Use a lock file to avoid exceptions due to concurrenct symlink creation

We have seen exceptions being raised from _update_root_symlink() on the
level of the sigstore-python library when multiple concurrent threads
were creating symlinks in this function for the same file (in a test
environment running tests concurrently). To avoid this issue, have each
thread open a lock file and create an exclusive lock on it to serialize
the access to the removal and creation of the symlink.

The reproducer for this issue, that should be run in 2 or more python
interpreters concurrently, looks like this:

from sigstore import sign
while True:
    sign.TrustedRoot.production()

Use fcntl.lockf-based locking for Linux and Mac and a different
implementation on Windows. The source originally comes from a
discussion on stockoverflow (link below).

Resolves: #2836
Link: https://stackoverflow.com/questions/489861/locking-a-file-in-python
Signed-off-by: Stefan Berger <[email protected]>

Author: stefanberger

tuf/ngclient/file_lock.py

@@ -0,0 +1,22 @@
+# Copyright 2020, New York University and the TUF contributors
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+"""Minimal file locking implementation.
+
+Source: https://stackoverflow.com/questions/489861/locking-a-file-in-python
+"""
+
+try:
+    # Posix based file locking (Linux, Ubuntu, MacOS, etc.)
+    #   Only allows locking on writable files, might cause
+    #   strange results for reading.
+    import fcntl
+    def lock_file(f):
+        if f.writable(): fcntl.lockf(f, fcntl.LOCK_EX)
+except ModuleNotFoundError:
+    # Windows file locking
+    import msvcrt
+    def file_size(f):
+        return os.path.getsize( os.path.realpath(f.name) )
+    def lock_file(f):
+        msvcrt.locking(f.fileno(), msvcrt.LK_RLCK, file_size(f))

tuf/ngclient/updater.py

@@ -66,6 +66,7 @@
 from tuf.api.metadata import Root, Snapshot, TargetFile, Targets, Timestamp
 from tuf.ngclient._internal.trusted_metadata_set import TrustedMetadataSet
 from tuf.ngclient.config import EnvelopeType, UpdaterConfig
+from tuf.ngclient.file_lock import lock_file
 from tuf.ngclient.urllib3_fetcher import Urllib3Fetcher
 
 if TYPE_CHECKING:
@@ -362,9 +363,12 @@ def _update_root_symlink(self) -> None:
         linkname = os.path.join(self._dir, "root.json")
         version = self._trusted_set.root.version
         current = os.path.join("root_history", f"{version}.root.json")
-        with contextlib.suppress(FileNotFoundError):
-            os.remove(linkname)
-        os.symlink(current, linkname)
+
+        with open(os.path.join(self._dir, current + ".lck"), "wb") as f:
+            lock_file(f)
+            with contextlib.suppress(FileNotFoundError):
+                os.remove(linkname)
+            os.symlink(current, linkname)
 
     def _load_root(self) -> None:
         """Load root metadata.