Merge pull request nexpy#261 from rayosborn:improve-encoding-detection

Improve-encoding-detection

Author: rayosborn

pyproject.toml

@@ -27,6 +27,7 @@ classifiers = [
 keywords = ["data format", "HDF5", "neutron scattering", "x-ray scattering"]
 requires-python = ">=3.9"
 dependencies = [
+    "chardet",
     "colored",
     "h5py",
     "hdf5plugin",

src/nexusformat/nexus/completer.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------
-# Copyright (c) 2019-2025, NeXpy Development Team.
+# Copyright (c) 2019-2026, NeXpy Development Team.
 #
 # Author: Paul Kienzle, Ray Osborn
 #

src/nexusformat/nexus/lock.py

@@ -1,5 +1,5 @@
 # -----------------------------------------------------------------------------
-# Copyright (c) 2019-2022, NeXpy Development Team.
+# Copyright (c) 2019-2026, NeXpy Development Team.
 #
 # Author: Paul Kienzle, Ray Osborn
 #

src/nexusformat/nexus/tree.py

@@ -266,16 +266,17 @@ def text(value):
     if isinstance(value, bytes):
         try:
             _text = value.decode(NX_CONFIG['encoding'])
-        except UnicodeDecodeError:
-            if NX_CONFIG['encoding'] == 'utf-8':
-                _text = value.decode('latin-1')
-            else:
-                _text = value.decode('utf-8')
+        except (UnicodeDecodeError, KeyError, LookupError):
+            import chardet
+            detected = chardet.detect(value)
+            encoding = detected['encoding']
+            if not encoding:
+                encoding = 'latin-1'
+            _text = value.decode(encoding, errors='replace')
     else:
         _text = str(value)
     return _text.replace('\x00', '').rstrip()
 
-
 def is_text(value):
     """
     Return True if the value represents text.

src/nexusformat/scripts/nexusformat.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2019-2021, NeXpy Development Team.
+# Copyright (c) 2019-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

src/nexusformat/scripts/nxcheck.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2025, NeXpy Development Team.
+# Copyright (c) 2025-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

src/nexusformat/scripts/nxconsolidate.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2022, NeXpy Development Team.
+# Copyright (c) 2022-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

src/nexusformat/scripts/nxdir.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2019-2021, NeXpy Development Team.
+# Copyright (c) 2019-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

src/nexusformat/scripts/nxduplicate.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2019-2021, NeXpy Development Team.
+# Copyright (c) 2019-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

src/nexusformat/scripts/nxinspect.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2025, NeXpy Development Team.
+# Copyright (c) 2025-2026, NeXpy Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #