Fix copy hardlinks in shared packages (BobBuildTool#680)

Author: jkloetzke

pym/bob/share.py

@@ -10,12 +10,14 @@
 import os, os.path
 import json
 import shutil
+import stat
 import sys
 import tempfile
 
 warnRepoSize = WarnOnce("The shared repository is over its quota. Run 'bob clean --shared' to free disk space!")
 warnGcDidNotHelp = WarnOnce("The automatic garbage collection of the shared repository was unable to free enough space. Run 'bob clean --shared' manually.")
 warnNoShareConfigured = Warn("No shared directory configured! Nothing cleaned.")
+warnEscapedHardLink = Warn("The file has hard links outside the workspace! Review the packageScript to fix the problem.")
 
 if sys.platform == 'win32':
     import msvcrt
@@ -57,6 +59,52 @@ def __exit__(self, exc_type, exc_value, traceback):
             self.fd.close()
 
 
+if isWindows():
+    class CopyMachine:
+        def __call__(self, src, dst):
+            return shutil.copy2(src, dst)
+        def postCopyCheck(self):
+            pass
+else:
+    class CopyMachine:
+        def __init__(self):
+            self.hard_links = {}
+
+        def __call__(self, src, dst):
+            if os.path.isdir(dst):
+                dst = os.path.join(dst, os.path.basename(src))
+
+            st = os.stat(src)
+            if st.st_nlink <= 1 or not self.__handle_hard_link(src, dst, st):
+                shutil.copyfile(src, dst)
+                shutil.copystat(src, dst)
+
+            return dst
+
+        def __handle_hard_link(self, src, dst, st):
+            key = (st.st_ino, st.st_dev)
+            file_rec = self.hard_links.get(key)
+            if file_rec is not None:
+                # We've seen this one... We assume that we don't cross mount
+                # points at destination!
+                existing, remaining = file_rec
+                os.link(existing, dst)
+                if remaining <= 1:
+                    del self.hard_links[key]
+                else:
+                    file_rec[1] -= 1
+                return True
+            else:
+                # First time we visited this hard-linked file. Remember with
+                # remaining hard links.
+                self.hard_links[key] = [dst, st.st_nlink - 1]
+                return False
+
+        def postCopyCheck(self):
+            for _, (name, _links) in self.hard_links.items():
+                warnEscapedHardLink.show(name)
+
+
 def sameWorkspace(link, sharePath):
     """Is the workspace shared and points to sharePath?
 
@@ -220,11 +268,14 @@ def installSharedPackage(self, workspace, buildId, sharedHash, mayMove):
                 # Might not exist if workspace was emtpy
                 if os.path.exists(cacheBinSrc):
                     shutil.copyfile(cacheBinSrc, cacheBinDst)
+
+                copyFun = CopyMachine()
                 if mayMove:
-                    shutil.move(workspace, tmpSharedPath)
+                    shutil.move(workspace, tmpSharedPath, copy_function=copyFun)
                 else:
                     shutil.copytree(workspace, os.path.join(tmpSharedPath, "workspace"),
-                        symlinks=True)
+                        symlinks=True, copy_function=copyFun)
+                copyFun.postCopyCheck()
 
                 # Cerify the result hash and count file system size. The user
                 # could have an incompatible file system at the destination.

test/unit/test_share.py

@@ -0,0 +1,76 @@
+# Bob Build Tool
+# Copyright (C) 2025  Jan Klötzke
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+from unittest import TestCase, skipIf
+from unittest.mock import MagicMock, patch
+import os
+import sys
+
+from bob.share import LocalShare
+from bob.utils import hashDirectory
+from bob.errors import BuildError
+
+class TestLocalShare(TestCase):
+
+    def setUp(self):
+        # create repo
+        self.repo = TemporaryDirectory()
+        self.share = LocalShare({ 'path' : self.repo.name })
+        self.pkg_tmp = TemporaryDirectory()
+        self.pkg = self.pkg_tmp.name
+        with open(os.path.join(self.pkg, "audit.json.gz"), "wb") as f:
+            pass
+        self.workspace = os.path.join(self.pkg, "workspace")
+        os.mkdir(self.workspace)
+
+    def tearDown(self):
+        self.pkg_tmp.cleanup()
+        self.repo.cleanup()
+
+    @skipIf(sys.platform.startswith("win"), "requires POSIX platform")
+    def testInstallHardLinks(self):
+        """Hard links are preserved when copying"""
+        with open(os.path.join(self.workspace, "a"), "wb") as f:
+            f.write(b'a')
+        os.link(os.path.join(self.workspace, "a"), os.path.join(self.workspace, "b"))
+        os.link(os.path.join(self.workspace, "a"), os.path.join(self.workspace, "c"))
+
+        with patch('bob.share.warnEscapedHardLink') as warning:
+            warning.show = MagicMock()
+            bid = b'a'*20
+            self.share.installSharedPackage(self.workspace, bid, hashDirectory(self.workspace), False)
+            warning.show.assert_not_called()
+
+        sharePath = self.share._LocalShare__buildPath(bid)
+        s1 = os.stat(os.path.join(sharePath, "workspace", "a"))
+        s2 = os.stat(os.path.join(sharePath, "workspace", "b"))
+        s3 = os.stat(os.path.join(sharePath, "workspace", "c"))
+        self.assertEqual(s1.st_nlink, 3)
+        self.assertEqual(s2.st_nlink, 3)
+        self.assertEqual(s3.st_nlink, 3)
+        self.assertEqual(s1.st_dev, s2.st_dev)
+        self.assertEqual(s1.st_ino, s2.st_ino)
+        self.assertEqual(s2.st_dev, s3.st_dev)
+        self.assertEqual(s2.st_ino, s3.st_ino)
+
+    @skipIf(sys.platform.startswith("win"), "requires POSIX platform")
+    def testInstallHardLinkOutsideWorkspace(self):
+        """Hard links outside the workspace create a warning"""
+        with open(os.path.join(self.pkg, "outside"), "wb") as f:
+            f.write(b'a')
+        os.link(os.path.join(self.pkg, "outside"), os.path.join(self.workspace, "file"))
+
+        with patch('bob.share.warnEscapedHardLink') as warning:
+            warning.show = MagicMock()
+            bid = b'a'*20
+            self.share.installSharedPackage(self.workspace, bid, hashDirectory(self.workspace), False)
+            warning.show.assert_called()
+
+    def testInstallCorrupted(self):
+        """Changes to the hash sum at the destination are detected"""
+        with self.assertRaises(BuildError):
+            self.share.installSharedPackage(self.workspace, b'a'*20, b'0'*20, False)
+        with self.assertRaises(BuildError):
+            self.share.installSharedPackage(self.workspace, b'a'*20, b'0'*20, True)