Merge pull request #38 from tskisner/support_large_buffers

Work around 32-bit limit on MPI buffer size

Author: keskitalo

src/pshmem/shmem.py

@@ -543,7 +543,16 @@ def set(self, data, offset=None, fromrank=0):
                     nodedata = np.zeros(datashape, dtype=self._dtype)
 
                 # Broadcast the data buffer
-                self._rankcomm.Bcast([nodedata, self._mpitype], root=fromnode)
+                nodedata = nodedata.reshape(-1, copy=False)
+                maxlen = 2**30
+                start = 0
+                while start < nodedata.size:
+                    stop = min(start + maxlen, nodedata.size)
+                    self._rankcomm.Bcast(
+                        [nodedata[start:stop], self._mpitype], root=fromnode
+                    )
+                    start = stop
+                nodedata = nodedata.reshape(datashape, copy=False)
 
                 # Now one process on every node has a copy of the data, and
                 # can copy it into the shared memory buffer.