missing manifest in 2 files

[vincelhx]

missing manifest for these 2 files :

notin = [
    '/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm3/SC50MB/GRD/2023/051/RCM3_OK2450914_PK2452956_1_SC50MB_20230220_011828_VV_VH_GRD',
    '/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm1/SCLND/GRD/2023/041/RCM1_OK2442646_PK2443937_1_SCLND_20230210_080341_VV_VH_GRD'
]
tree = safe_rcm.open_rcm(notin[1], chunks={})

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/mapping.py:143, in FSMap.__getitem__(self, key, default)
   142 try:
--> 143     result = self.fs.cat(k)
   144 except self.missing_exceptions:

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/spec.py:843, in AbstractFileSystem.cat(self, path, recursive, on_error, **kwargs)
   842 else:
--> 843     return self.cat_file(paths[0], **kwargs)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/spec.py:742, in AbstractFileSystem.cat_file(self, path, start, end, **kwargs)
   741 # explicitly set buffering off?
--> 742 with self.open(path, "rb", **kwargs) as f:
   743     if start is not None:

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/spec.py:1154, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
  1153 ac = kwargs.pop("autocommit", not self._intrans)
-> 1154 f = self._open(
  1155     path,
  1156     mode=mode,
  1157     block_size=block_size,
  1158     autocommit=ac,
  1159     cache_options=cache_options,
  1160     **kwargs,
  1161 )
  1162 if compression is not None:

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/implementations/local.py:183, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)
   182     self.makedirs(self._parent(path), exist_ok=True)
--> 183 return LocalFileOpener(path, mode, fs=self, **kwargs)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/implementations/local.py:287, in LocalFileOpener.__init__(self, path, mode, autocommit, fs, compression, **kwargs)
   286 self.blocksize = io.DEFAULT_BUFFER_SIZE
--> 287 self._open()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/implementations/local.py:292, in LocalFileOpener._open(self)
   291 if self.autocommit or "w" not in self.mode:
--> 292     self.f = open(self.path, mode=self.mode)
   293     if self.compression:

FileNotFoundError: [Errno 2] No such file or directory: '/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm1/SCLND/GRD/2023/041/RCM1_OK2442646_PK2443937_1_SCLND_20230210_080341_VV_VH_GRD/support/schemas/rcm_prod_manifest.xsd'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/api.py:79, in open_rcm(url, backend_kwargs, manifest_ignores, **dataset_kwargs)
    78 try:
---> 79     declared_files = read_manifest(mapper, "manifest.safe")
    80 except (FileNotFoundError, KeyError):

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/manifest.py:43, in read_manifest(mapper, path)
    17 structure = {
    18     "/dataObjectSection/dataObject": compose_left(
    19         curry(
  (...)
    40     ),
    41 }
---> 43 manifest = read_xml(mapper, path)
    45 return list(concat(func(query(path, manifest)) for path, func in structure.items()))

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/xml.py:82, in read_xml(mapper, path)
    79 schema_path = posixpath.normpath(
    80     posixpath.join(posixpath.dirname(path), schema_path_)
    81 )
---> 82 schema = open_schema(mapper, schema_path)
    84 decoded = schema.decode(tree)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/xml.py:65, in open_schema(mapper, schema)
    47 """fsspec-compatible way to open remote schema files
    48 
    49 Parameters
  (...)
    63     The opened schema object
    64 """
---> 65 paths = schema_paths(mapper, schema)
    66 preprocessed = [remove_includes(mapper[p].decode()) for p in paths]

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/xml.py:35, in schema_paths(mapper, root_schema)
    33 visited.append(path)
---> 35 text = mapper[path].decode()
    36 includes = extract_includes(text)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/fsspec/mapping.py:147, in FSMap.__getitem__(self, key, default)
   146         return default
--> 147     raise KeyError(key)
   148 return result

KeyError: 'support/schemas/rcm_prod_manifest.xsd'

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Cell In[40], line 3
     1 import safe_rcm
----> 3 tree = safe_rcm.open_rcm(notin[2], chunks={})

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/api.py:81, in open_rcm(url, backend_kwargs, manifest_ignores, **dataset_kwargs)
    79     declared_files = read_manifest(mapper, "manifest.safe")
    80 except (FileNotFoundError, KeyError):
---> 81     raise ValueError(
    82         "cannot find the `manifest.safe` file. Are you sure this is a SAFE dataset?"
    83     )
    85 missing_files = [
    86     path
    87     for path in declared_files
    88     if not ignored_file(path, manifest_ignores)
    89     and not mapper.fs.exists(mapper._key_to_str(path))
    90 ]
    91 if missing_files:

ValueError: cannot find the `manifest.safe` file. Are you sure this is a SAFE dataset?

[keewis]

the entire support directory with the schemas and the directory containing the metadata are missing (manifest.safe exists, though). So even if we fixed that, the reader would still fail. To fix that, you will have to re-download those particular scenes (or ask the one who downloaded it do that for you).

That said, I'll keep this open because the error message does not really help: the manifest exists, it's the schema that's missing.

[agrouaze]

The products were corrupted on IFREMER archive, they have been re download and now opening them is working properly:

notin = [
   ...:     '/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm3/SC50MB/GRD/2023/051/RCM3_OK2450914_PK2452956_1_SC50MB_20230220_011828_VV_VH_GRD',
   ...:     '/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm1/SCLND/GRD/2023/041/RCM1_OK2442646_PK2443937_1_SCLND_20230210_080341_VV_VH_GRD'
   ...: ]
   ...: tree = safe_rcm.open_rcm(notin[1], chunks={})
/opt/conda-envs/dev/lib/python3.10/site-packages/safe_rcm/product/reader.py:59: UserWarning: Converting non-nanosecond precision datetime values to nanosecond precision. This behavior can eventually be relaxed in xarray, as it is an artifact from pandas which is now beginning to support non-nanosecond precision values. This warning is caused by passing non-nanosecond np.datetime64 or np.timedelta64 values to the DataArray or Variable constructor; it can be silenced by converting the values to nanosecond precision ahead of time.
  {"timeStamp": ds["timeStamp"].astype("datetime64")}
/opt/conda-envs/dev/lib/python3.10/site-packages/safe_rcm/product/reader.py:68: UserWarning: Converting non-nanosecond precision datetime values to nanosecond precision. This behavior can eventually be relaxed in xarray, as it is an artifact from pandas which is now beginning to support non-nanosecond precision values. This warning is caused by passing non-nanosecond np.datetime64 or np.timedelta64 values to the DataArray or Variable constructor; it can be silenced by converting the values to nanosecond precision ahead of time.
  {"timeStamp": ds["timeStamp"].astype("datetime64")}

In [4]: tree = safe_rcm.open_rcm(notin[0], chunks={})
/opt/conda-envs/dev/lib/python3.10/site-packages/safe_rcm/product/reader.py:59: UserWarning: Converting non-nanosecond precision datetime values to nanosecond precision. This behavior can eventually be relaxed in xarray, as it is an artifact from pandas which is now beginning to support non-nanosecond precision values. This warning is caused by passing non-nanosecond np.datetime64 or np.timedelta64 values to the DataArray or Variable constructor; it can be silenced by converting the values to nanosecond precision ahead of time.
  {"timeStamp": ds["timeStamp"].astype("datetime64")}
/opt/conda-envs/dev/lib/python3.10/site-packages/safe_rcm/product/reader.py:68: UserWarning: Converting non-nanosecond precision datetime values to nanosecond precision. This behavior can eventually be relaxed in xarray, as it is an artifact from pandas which is now beginning to support non-nanosecond precision values. This warning is caused by passing non-nanosecond np.datetime64 or np.timedelta64 values to the DataArray or Variable constructor; it can be silenced by converting the values to nanosecond precision ahead of time.
  {"timeStamp": ds["timeStamp"].astype("datetime64")}

[vincelhx]

Thanks, i close the issue.

[keewis]

reopening to keep track of improving the error message