CA-424055: NUMA: avoid using up the entire memory on node0

Versions of Xen without per-node claim support won't allow using up
all the memory from node0 (it'll move some of its memory to other nodes).
On versions of Xen with per-node claim support it'll respect the claim,
but it is undesirable to completely run out of memory on node0, since some
devices may require the low 4GiB of RAM for DMA.

Xen would reserve 2^32 on node0, and only use it as last resort.
However we don't know how much of that has been used up already.
For now reserve just 2^31 by default, and make this configurable in
xenopsd.conf.

Signed-off-by: Edwin Török <edwin.torok@citrix.com>

Author: edwintorok

ocaml/xenopsd/lib/xenopsd.ml

@@ -72,6 +72,8 @@ let numa_placement_compat = ref true
 
 let numa_best_effort_prio_mem_only = ref false
 
+let numa_reserve_node0_dmaheap_bits = ref 31
+
 (* O(N^2) operations, until we get a xenstore cache, so use a small number here *)
 let vm_guest_agent_xenstore_quota = ref 128
 
@@ -282,6 +284,12 @@ let options =
        filtered NUMA nodes based on available memory. Only use if there are \
        issues with the new best effort policy"
     )
+  ; ( "numa-reserve-node0-dmaheap-bits"
+    , Arg.Int (fun x -> numa_reserve_node0_dmaheap_bits := x)
+    , (fun () -> string_of_int !numa_reserve_node0_dmaheap_bits)
+    , "Reserve 2^N bytes on node0 in the NUMA planner, similar to how Xen would\n\
+      \  protect the low 4GiB of the memory for the DMA heap"
+    )
   ; ( "pci-quarantine"
     , Arg.Bool (fun b -> pci_quarantine := b)
     , (fun () -> string_of_bool !pci_quarantine)

ocaml/xenopsd/xc/domain.ml

@@ -1099,6 +1099,20 @@ let numa_mutex = Mutex.create ()
 (* protected by numa_mutex *)
 let numa_resources = ref NUMAResources.empty
 
+let numa_reserve_node0 mem =
+  let reserved = Int64.shift_left 1L !Xenopsd.numa_reserve_node0_dmaheap_bits in
+  let mem' = Int64.(max 0L (sub mem reserved)) in
+  (* Prefer nodes other than node0 when node0 is low on memory.
+     This is reserved for the DMA heap in Xen, and even if we try to make a VM
+     use it up, Xen will move some of its memory to other nodes to avoid
+     running out.
+     Without the claim API we have no way of forcing or querying this
+     (and even with the claim API it'd be desirable to avoid running out on
+     node0).
+   *)
+  D.debug "numa_reserve_node0, reducing free memory: %Lu -> %Lu" mem mem' ;
+  mem'
+
 let numa_init () =
   let xcext = Xenctrlext.get_handle () in
   let host = Lazy.force numa_hierarchy in
@@ -1129,7 +1143,19 @@ let numa_placement domid ~vcpus ~cores ~memory ~required_free affinity =
       let numa_meminfo = (numainfo xcext).memory |> Array.to_seq in
       let nodes =
         Seq.map2
-          (fun node m -> (node, NUMA.resource host node ~memory:m.memfree))
+          (fun node m ->
+            ( node
+            , let (NUMA.Node nodeid) = node in
+              let memory = m.memfree in
+              let memory =
+                if nodeid = 0 then
+                  numa_reserve_node0 memory
+                else
+                  memory
+              in
+              NUMA.resource host node ~memory
+            )
+          )
           (NUMA.nodes host) numa_meminfo
       in
       let vm = NUMARequest.make ~memory ~vcpus ~cores in