When working around kernel bugs writing strange mappings via /proc/.../mem, only fix the mappings that actually overlap the area we're going to write.

Trying to fix the whole mapping at once triggers `ENOMEM` on most kernel versions,
e.g. 6.15, if the mapping is `PROT_NONE` `MAP_PRIVATE`.

Resolves #3976

Author: rocallahan

CMakeLists.txt

@@ -1175,6 +1175,7 @@ set(BASIC_TESTS
   mlock
   mlock_madvise
   mmap_adjacent_to_rr_usage
+  mmap_huge
   mmap_private
   mmap_private_grow_under_map
   mmap_recycle

src/Task.cc

@@ -3186,12 +3186,15 @@ void Task::read_bytes_helper(remote_ptr<void> addr, ssize_t buf_size, void* buf,
 }
 
 /**
- * This function exists to work around
+ * This function was initially created to work around
  * https://bugzilla.kernel.org/show_bug.cgi?id=99101.
  * On some kernels pwrite() to /proc/.../mem fails when writing to a region
- * that's PROT_NONE.
- * Also, writing through MAP_SHARED readonly mappings fails (even if the
- * file was opened read-write originally), so we handle that here too.
+ * that's PROT_NONE. Actually this was fixed in kernel 4.8 but we still
+ * support 4.7 for now and there's no point in bumping the version
+ * requirement just to avoid this, especially because...
+ * Writing through MAP_SHARED readonly mappings fails (even if the
+ * file was opened read-write originally), so we handle that here too. This
+ * seems to be an issue for all kernel versions up to the present day.
  */
 static ssize_t safe_pwrite64(Task* t, const void* buf, ssize_t buf_size,
                              remote_ptr<void> addr) {
@@ -3205,9 +3208,14 @@ static ssize_t safe_pwrite64(Task* t, const void* buf, ssize_t buf_size,
       continue;
     }
     if (!(m.map.prot() & PROT_READ) || (m.map.flags() & MAP_SHARED)) {
-      mappings_to_fix.push_back(m.map);
+      // Limit the mapping change to the region we're actually going to write to.
+      // Kernel 6.15 reports ENOMEM trying mprotect PROT_WRITE colossal
+      // MAP_PRIVATE mappings. For some reason MAP_SHARED is OK...
+      auto start = floor_page_size(addr);
+      auto end = ceil_page_size(addr + buf_size);
+      mappings_to_fix.push_back(m.map.subrange(start, end));
     }
-  };
+  }
 
   if (mappings_to_fix.empty()) {
     return pwrite_all_fallible(t->vm()->mem_fd(), buf, buf_size, addr.as_int());

src/test/mmap_huge.c

@@ -0,0 +1,34 @@
+/* -*- Mode: C; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
+
+#include "util.h"
+
+int main(void) {
+  if (sizeof(void*) == 4) {
+    atomic_puts("Skipping test on 32-bit");
+    atomic_puts("EXIT-SUCCESS");
+    return 0;
+  }
+
+  int fd = memfd_create("rr_test", 0);
+  test_assert(fd >= 0);
+  uint64_t len = 1LL << 40;
+  int ret = ftruncate(fd, len);
+  test_assert(ret == 0);
+  ret = write(fd, "x", 1);
+  test_assert(ret == 1);
+
+  int flags[] = { PROT_NONE, PROT_READ };
+  int mode[] = { MAP_PRIVATE, MAP_SHARED };
+  for (int i = 0; i < 2; ++i) {
+    for (int j = 0; j < 2; ++j) {
+      void* p = mmap(NULL, len, flags[i], mode[j], fd, 0);
+      test_assert(p != MAP_FAILED);
+      ret = munmap(p, len);
+      test_assert(ret == 0);
+    }
+  }
+
+  atomic_puts("EXIT-SUCCESS");
+
+  return 0;
+}