Use scandir in DirectoryStore.getsize (#431)

* Require `scandir` for Python pre-3.5

As `scandir` was added to the `os` module in Python 3.5, require the
backport package for `scandir` is installed for versions prior to
that (e.g. Python 2.7).

* Import `scandir` on Python 2/3

* Use `scandir` to determine directory size quickly

* Note use of scandir in DirectoryStore's getsize

* Use double backticks in release note's inline code

* Use escaped whitespace after double backticks

Author: jakirkham

docs/release.rst

@@ -6,13 +6,19 @@ Release notes
 2.3.2
 -----
 
+Enhancements
+~~~~~~~~~~~~
+
+* Use ``scandir`` in ``DirectoryStore``'s ``getsize`` method.
+  By :user:`John Kirkham <jakirkham>`; :issue:`431`
+
 Bug fixes
 ~~~~~~~~~
 
 * Add and use utility functions to simplify reading and writing JSON.
   By :user:`John Kirkham <jakirkham>`; :issue:`429`, :issue:`430`
 
-* Fix `collections`'s `DeprecationWarning`s.
+* Fix ``collections``'s ``DeprecationWarning``\ s.
   By :user:`John Kirkham <jakirkham>`; :issue:`432`
 
 

setup.py

@@ -17,6 +17,8 @@
     'numcodecs>=0.6.2',
 ]
 
+if sys.version_info < (3, 5):
+    dependencies.append('scandir')
 if sys.version_info < (3, 3) and sys.platform == "win32":
     dependencies.append('pyosreplace')
 

zarr/compat.py

@@ -21,6 +21,7 @@ def OrderedDict_move_to_end(od, key):
         od[key] = od.pop(key)
 
     from collections import Mapping, MutableMapping
+    from scandir import scandir
 
 
 else:  # pragma: py2 no cover
@@ -35,3 +36,4 @@ def OrderedDict_move_to_end(od, key):
         od.move_to_end(key)
 
     from collections.abc import Mapping, MutableMapping
+    from os import scandir

zarr/storage.py

@@ -37,7 +37,7 @@
                        normalize_storage_path, buffer_size,
                        normalize_fill_value, nolock, normalize_dtype)
 from zarr.meta import encode_array_metadata, encode_group_metadata
-from zarr.compat import PY2, MutableMapping, OrderedDict_move_to_end
+from zarr.compat import PY2, MutableMapping, OrderedDict_move_to_end, scandir
 from numcodecs.registry import codec_registry
 from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray
 from zarr.errors import (err_contains_group, err_contains_array, err_bad_compressor,
@@ -846,12 +846,10 @@ def getsize(self, path=None):
         if os.path.isfile(fs_path):
             return os.path.getsize(fs_path)
         elif os.path.isdir(fs_path):
-            children = os.listdir(fs_path)
             size = 0
-            for child in children:
-                child_fs_path = os.path.join(fs_path, child)
-                if os.path.isfile(child_fs_path):
-                    size += os.path.getsize(child_fs_path)
+            for child in scandir(fs_path):
+                if child.is_file():
+                    size += child.stat().st_size
             return size
         else:
             return 0