Add chardet package to handle text decoding failures

Author: rayosborn

pyproject.toml

@@ -27,6 +27,7 @@ classifiers = [
 keywords = ["data format", "HDF5", "neutron scattering", "x-ray scattering"]
 requires-python = ">=3.9"
 dependencies = [
+    "chardet",
     "colored",
     "h5py",
     "hdf5plugin",

src/nexusformat/nexus/tree.py

@@ -218,6 +218,7 @@
 
 import h5py as h5
 import numpy as np
+from typeguard import value
 
 from .. import __version__ as nxversion
 from .lock import NXLock, NXLockException
@@ -266,16 +267,17 @@ def text(value):
     if isinstance(value, bytes):
         try:
             _text = value.decode(NX_CONFIG['encoding'])
-        except UnicodeDecodeError:
-            if NX_CONFIG['encoding'] == 'utf-8':
-                _text = value.decode('latin-1')
-            else:
-                _text = value.decode('utf-8')
+        except (UnicodeDecodeError, KeyError, LookupError):
+            import chardet
+            detected = chardet.detect(value)
+            encoding = detected['encoding']
+            if not encoding:
+                encoding = 'latin-1'
+            _text = value.decode(encoding, errors='replace')
     else:
         _text = str(value)
     return _text.replace('\x00', '').rstrip()
 
-
 def is_text(value):
     """
     Return True if the value represents text.