Fix fetchdata layout regression and add event_files output mode

Author: claudiodsf

CHANGELOG.md

@@ -15,6 +15,14 @@ Copyright (c) 2022-2026 Claudio Satriano <satriano@ipgp.fr>
   background catalog updates, inspect daemon status, and
   install/uninstall `launchd` or `systemd` service units for periodic
   `updatedb`, event-details, and waveform-fetch cycles.
+- New `fetchdata_layout` configuration option to control how fetched data is
+  organized:
+  - `event_dirs` (default):
+    `events/<evid>/<evid>.xml`, `events/<evid>/waveforms/`,
+    `events/<evid>/stations/`
+  - `event_files`:
+    `events/<evid>/event.xml`, `events/<evid>/event.mseed`,
+    `events/<evid>/stations.xml`
 
 ### Fixed
 
@@ -26,6 +34,9 @@ Copyright (c) 2022-2026 Claudio Satriano <satriano@ipgp.fr>
   plotting/data dependencies choice: if plotting extras are not already
   installed, update no longer adds them implicitly (applies to release and
   `--git` update paths).
+- Fixed `seiscat fetchdata` directory layout regression where `waveforms/` and
+  `stations/` could be created as top-level directories instead of
+  per-event subdirectories.
 
 ## [0.9.4] - 2026-04-23
 

seiscat/config/config.py

@@ -104,7 +104,9 @@ def _validate_config(config_obj):
 
 # Keys whose values are file/directory paths that should be resolved
 # relative to the config file's directory when they are not absolute.
-_PATH_KEYS = ('db_file', 'event_dir', 'waveform_dir', 'station_dir')
+# waveform_dir and station_dir are event-subdirectory names and must stay
+# relative to each per-event folder.
+_PATH_KEYS = ('db_file', 'event_dir')
 
 
 def _resolve_path_keys(config_obj, config_file):

seiscat/config/configspec.conf

@@ -57,6 +57,16 @@ fdsn_providers_passwords = force_list(default=None)
 # Directory to store event data. For each event, an event directory named
 # after the event ID will be created inside this directory.
 event_dir = string(default=events)
+# Fetchdata output layout:
+# - event_dirs: legacy layout
+#     events/<evid>/<evid>.xml
+#     events/<evid>/<waveform_dir>/*.mseed
+#     events/<evid>/<station_dir>/*.xml
+# - event_files: bundled layout
+#     events/<evid>/event.xml
+#     events/<evid>/event.mseed
+#     events/<evid>/stations.xml
+fetchdata_layout = string(default=event_dirs)
 # Directory to store waveform data.
 # It will be created inside the event directory.
 waveform_dir = string(default=waveforms)

seiscat/fetchdata/event_details.py

@@ -17,6 +17,7 @@
 from ..database.dbfunctions import read_events_from_db
 from ..sources.fdsnws import open_fdsn_connection
 from ..utils import ExceptionExit
+from .event_waveforms_utils import get_event_layout_paths
 
 
 def _get_events(client, evid):
@@ -88,9 +89,10 @@ def fetch_event_details(config):
         # .get() returns None if 'raw_evid' column doesn't exist
         raw_evid = event.get('raw_evid')
         print(f'{evid}:', end=' ')
-        evid_dir = pathlib.Path(event_dir / f'{evid}')
+        paths = get_event_layout_paths(config, evid)
+        evid_dir = pathlib.Path(paths['evid_dir'])
         evid_dir.mkdir(parents=True, exist_ok=True)
-        outfile = evid_dir / f'{evid}.xml'
+        outfile = paths['event_xml_file']
         if not overwrite_existing and outfile.exists():
             print(f'{outfile} exists, skipping')
             continue

seiscat/fetchdata/event_waveforms_utils.py

@@ -75,6 +75,136 @@ def prefer_high_sampling_rate(waveform_dir, logger=None):
             file.unlink()
 
 
+def get_fetchdata_layout(config):
+    """
+    Get the fetchdata output layout.
+
+    Supported values:
+    - ``event_dirs``: legacy layout with per-event waveform/station folders
+    - ``event_files``: bundled per-event files (event.mseed/stations.xml)
+
+    :param config: config object
+    :type config: dict
+    :return: normalized layout name
+    :rtype: str
+    """
+    layout = str(config.get('fetchdata_layout', 'event_dirs')).strip().lower()
+    if layout in ('legacy', 'event_dirs', 'event_dir'):
+        return 'event_dirs'
+    if layout in ('event_files', 'bundled'):
+        return 'event_files'
+    return 'event_dirs'
+
+
+def get_event_xml_file(evid_dir, evid):
+    """
+    Return the event QuakeML file path for an event directory.
+
+    Tries ``{evid}.xml`` first, then ``event.xml``.
+
+    :param evid_dir: event directory
+    :type evid_dir: pathlib.Path
+    :param evid: event ID
+    :type evid: str
+    :return: event xml file path or None
+    :rtype: pathlib.Path or None
+    """
+    evid_dir = pathlib.Path(evid_dir)
+    candidates = (
+        evid_dir / f'{evid}.xml',
+        evid_dir / 'event.xml',
+    )
+    for xml_file in candidates:
+        if xml_file.exists():
+            return xml_file
+    return None
+
+
+def get_event_layout_paths(config, evid):
+    """
+    Build event-specific paths for the configured fetchdata layout.
+
+    :param config: config object
+    :type config: dict
+    :param evid: event ID
+    :type evid: str
+    :return: dictionary with layout and file/directory paths
+    :rtype: dict
+    """
+    event_dir = pathlib.Path(config['event_dir'])
+    evid_dir = event_dir / f'{evid}'
+    layout = get_fetchdata_layout(config)
+    if layout == 'event_files':
+        return {
+            'layout': layout,
+            'evid_dir': evid_dir,
+            'waveform_dir': evid_dir / '.waveforms',
+            'station_dir': evid_dir / '.stations',
+            'event_xml_file': evid_dir / 'event.xml',
+            'waveform_file': evid_dir / 'event.mseed',
+            'station_file': evid_dir / 'stations.xml',
+        }
+    return {
+        'layout': layout,
+        'evid_dir': evid_dir,
+        'waveform_dir': evid_dir / config['waveform_dir'],
+        'station_dir': evid_dir / config['station_dir'],
+        'event_xml_file': evid_dir / f'{evid}.xml',
+        'waveform_file': None,
+        'station_file': None,
+    }
+
+
+def bundle_waveforms_to_mseed(waveform_dir, outfile):
+    """
+    Bundle all waveform miniSEED files from a directory into one file.
+
+    :param waveform_dir: source waveform directory
+    :type waveform_dir: pathlib.Path
+    :param outfile: output miniSEED file
+    :type outfile: pathlib.Path
+    :return: True if output file was written, False otherwise
+    :rtype: bool
+    """
+    from obspy import Stream, read
+    waveform_dir = pathlib.Path(waveform_dir)
+    outfile = pathlib.Path(outfile)
+    stream = Stream()
+    for mseed_file in sorted(waveform_dir.glob('*.mseed')):
+        stream += read(str(mseed_file))
+    if len(stream) == 0:
+        return False
+    stream.write(str(outfile), format='MSEED')
+    return True
+
+
+def bundle_stations_to_xml(station_dir, outfile):
+    """
+    Bundle all station XML files from a directory into one StationXML file.
+
+    :param station_dir: source station XML directory
+    :type station_dir: pathlib.Path
+    :param outfile: output StationXML file
+    :type outfile: pathlib.Path
+    :return: True if output file was written, False otherwise
+    :rtype: bool
+    """
+    from obspy import read_inventory
+    station_dir = pathlib.Path(station_dir)
+    outfile = pathlib.Path(outfile)
+    inv = None
+    for station_file in sorted(station_dir.glob('*.xml')):
+        _inv = read_inventory(str(station_file))
+        if inv is None:
+            inv = _inv
+        else:
+            inv += _inv
+    if inv is None:
+        return False
+    inv.write(str(outfile), format='STATIONXML')
+    return True
+
+
 def check_station(station, station_codes):
     """
     Check if a station matches the specified station codes.
@@ -116,8 +246,8 @@ def get_picked_station_codes(evid_dir, evid):
     """
     import warnings
     from obspy import read_events
-    xml_file = pathlib.Path(evid_dir) / f'{evid}.xml'
-    if not xml_file.exists():
+    xml_file = get_event_xml_file(evid_dir, evid)
+    if xml_file is None:
         return None
     with warnings.catch_warnings():
         warnings.simplefilter('ignore')

seiscat/fetchdata/mass_downloader.py

@@ -12,11 +12,18 @@
 import sys
 import pathlib
 import logging
+import shutil
 from obspy.clients.fdsn import Client
 from obspy.clients.fdsn.mass_downloader import (
     CircularDomain, Restrictions, MassDownloader
 )
-from .event_waveforms_utils import prefer_high_sampling_rate, get_picked_station_codes
+from .event_waveforms_utils import (
+    prefer_high_sampling_rate,
+    get_event_layout_paths,
+    get_event_xml_file,
+    bundle_waveforms_to_mseed,
+    bundle_stations_to_xml,
+)
 from ..utils import ExceptionExit
 mdl_logger = logging.getLogger('obspy.clients.fdsn.mass_downloader')
 
@@ -234,10 +241,13 @@ def _build_station_restriction(
     if picked_stations_only:
         picked = get_picked_station_codes(evid_dir, evid)
         if picked is None:
+            event_xml = get_event_xml_file(evid_dir, evid)
+            if event_xml is None:
+                event_xml = pathlib.Path(evid_dir) / f'{evid}.xml'
             mdl_logger.warning(
                 'picked_stations_only is True but no event QuakeML file '
                 'found at %s. Ignoring pick-based station selection.',
-                evid_dir / f'{evid}.xml'
+                event_xml
             )
             # Fall through to plain station_codes restriction (or None)
             if not station_codes:
@@ -298,11 +308,11 @@ def mass_download_waveforms(config, event):
     channel_codes = config['channel_codes']
     station_codes = config['station_codes']
     picked_stations_only = config['picked_stations_only']
-    event_dir = pathlib.Path(config['event_dir'])
-    evid_dir = event_dir / f'{evid}'
-    waveform_dir = pathlib.Path(evid_dir / config['waveform_dir'])
+    paths = get_event_layout_paths(config, evid)
+    evid_dir = paths['evid_dir']
+    waveform_dir = pathlib.Path(paths['waveform_dir'])
     waveform_dir.mkdir(parents=True, exist_ok=True)
-    station_dir = pathlib.Path(evid_dir / config['station_dir'])
+    station_dir = pathlib.Path(paths['station_dir'])
     station_dir.mkdir(parents=True, exist_ok=True)
 
     _set_mdl_logger(evid)
@@ -361,6 +371,23 @@ def mass_download_waveforms(config, event):
         )
     if config['prefer_high_sampling_rate']:
         prefer_high_sampling_rate(waveform_dir, mdl_logger)
+    if paths['layout'] == 'event_files':
+        waveforms_written = bundle_waveforms_to_mseed(
+            waveform_dir, paths['waveform_file'])
+        stations_written = bundle_stations_to_xml(
+            station_dir, paths['station_file'])
+        shutil.rmtree(waveform_dir, ignore_errors=True)
+        shutil.rmtree(station_dir, ignore_errors=True)
+        if waveforms_written:
+            mdl_logger.info(
+                'Bundled waveforms saved to %s',
+                paths['waveform_file']
+            )
+        if stations_written:
+            mdl_logger.info(
+                'Bundled stations saved to %s',
+                paths['station_file']
+            )
     _info_msg = f'Waveforms and station metadata saved to {evid_dir}'
     mdl_logger.info(_info_msg)
     _unset_mdl_logger()

seiscat/fetchdata/sds.py

@@ -11,9 +11,15 @@
 """
 import pathlib
 import re
+import shutil
 from obspy.clients.filesystem.sds import Client
 from .event_waveforms_utils import (
-    prefer_high_sampling_rate, check_station, get_picked_station_codes
+    prefer_high_sampling_rate,
+    check_station,
+    get_picked_station_codes,
+    get_event_layout_paths,
+    get_event_xml_file,
+    bundle_waveforms_to_mseed,
 )
 
 
@@ -76,9 +82,9 @@ def fetch_sds_waveforms(config, event, client):
     :type client: obspy.clients.filesystem.sds.Client
     """
     evid = event['evid']
-    event_dir = pathlib.Path(config['event_dir'])
-    evid_dir = event_dir / f'{evid}'
-    waveform_dir = pathlib.Path(evid_dir / config['waveform_dir'])
+    paths = get_event_layout_paths(config, evid)
+    evid_dir = paths['evid_dir']
+    waveform_dir = pathlib.Path(paths['waveform_dir'])
     waveform_dir.mkdir(parents=True, exist_ok=True)
     seconds_before = config['seconds_before_origin']
     seconds_after = config['seconds_after_origin']
@@ -92,9 +98,12 @@ def fetch_sds_waveforms(config, event, client):
     if picked_stations_only:
         picked_stations = get_picked_station_codes(evid_dir, evid)
         if picked_stations is None:
+            event_xml = get_event_xml_file(evid_dir, evid)
+            if event_xml is None:
+                event_xml = evid_dir / f'{evid}.xml'
             print(
                 f'{evid}: picked_stations_only is True but no event QuakeML '
-                f'file found at {evid_dir / f"{evid}.xml"}. '
+                f'file found at {event_xml}. '
                 'Ignoring pick-based station selection.'
             )
         elif not picked_stations:
@@ -136,4 +145,8 @@ def fetch_sds_waveforms(config, event, client):
         print(f'  {outfile} written')
     if config['prefer_high_sampling_rate']:
         prefer_high_sampling_rate(waveform_dir)
+    if paths['layout'] == 'event_files':
+        if bundle_waveforms_to_mseed(waveform_dir, paths['waveform_file']):
+            print(f'  {paths["waveform_file"]} written')
+        shutil.rmtree(waveform_dir, ignore_errors=True)
     print()

tests/test_config_paths.py

@@ -0,0 +1,38 @@
+# -*- coding: utf8 -*-
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""
+Test config path resolution rules.
+"""
+import os
+import pathlib
+import tempfile
+import unittest
+from seiscat.config.config import parse_configspec, read_config
+
+
+class TestConfigPathResolution(unittest.TestCase):
+    """Test resolution of relative paths in config files."""
+
+    def test_event_dir_resolved_wave_station_kept_relative(self):
+        """event_dir is absolutized, waveform/station names remain relative."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cfg_path = pathlib.Path(tmpdir) / 'seiscat.conf'
+            cfg_path.write_text(
+                '\n'.join([
+                    'db_file = seiscat_db.sqlite',
+                    'event_dir = events',
+                    'waveform_dir = waveforms',
+                    'station_dir = stations',
+                ])
+            )
+            config = read_config(str(cfg_path), parse_configspec())
+            self.assertEqual(
+                config['event_dir'],
+                os.path.join(tmpdir, 'events')
+            )
+            self.assertEqual(config['waveform_dir'], 'waveforms')
+            self.assertEqual(config['station_dir'], 'stations')
+
+
+if __name__ == '__main__':
+    unittest.main()