Fix mypy errors in Ray/Modin modules

- Remove unused type: ignore comment in _utils.py
- Fix ObjectRef type annotation for from_arrow_refs call
- Remove FastFileMetadataProvider (removed from Ray)

Author: kukushking

awswrangler/distributed/ray/modin/_utils.py

@@ -37,7 +37,7 @@ def _ray_dataset_from_df(df: pd.DataFrame | modin_pd.DataFrame) -> Dataset:
     if isinstance(df, modin_pd.DataFrame):
         return from_modin(df)  # type: ignore[no-any-return]
     if isinstance(df, pd.DataFrame):
-        return from_pandas(df)  # type: ignore[no-any-return]
+        return from_pandas(df)
     raise ValueError(f"Unknown DataFrame type: {type(df)}")
 
 
@@ -85,7 +85,7 @@ def _is_not_empty(table: pa.Table) -> Any:
         )
 
     ref_rows: list[bool] = ray_get([_is_not_empty(arrow_ref) for arrow_ref in arrow_refs])
-    refs: list[Callable[..., Any]] = [ref for ref_rows, ref in zip(ref_rows, arrow_refs) if ref_rows]
+    refs: list[ObjectRef[Any]] = [ref for ref_rows, ref in zip(ref_rows, arrow_refs) if ref_rows]  # type: ignore[misc]
     return _to_modin(
         dataset=ray.data.from_arrow_refs(refs) if len(refs) > 0 else ray.data.from_arrow([pa.Table.from_arrays([])]),
         to_pandas_kwargs=kwargs,

awswrangler/distributed/ray/modin/s3/_read_orc.py

@@ -8,7 +8,6 @@
 import modin.pandas as pd
 import pyarrow as pa
 from ray.data import read_datasource
-from ray.data.datasource.file_meta_provider import FastFileMetadataProvider
 
 from awswrangler import _data_types
 from awswrangler.distributed.ray.datasources import ArrowORCDatasource
@@ -39,7 +38,6 @@ def _read_orc_distributed(
         use_threads=use_threads,
         schema=schema,
         arrow_orc_args={"columns": columns},
-        meta_provider=FastFileMetadataProvider(),
     )
     ray_dataset = read_datasource(
         datasource,

awswrangler/distributed/ray/modin/s3/_read_parquet.py

@@ -8,7 +8,6 @@
 import pyarrow as pa
 from ray.data import read_datasource
 from ray.data._internal.datasource.parquet_datasource import ParquetDatasource
-from ray.data.datasource.file_meta_provider import FastFileMetadataProvider
 
 from awswrangler.distributed.ray.datasources import ArrowParquetBaseDatasource
 from awswrangler.distributed.ray.modin._utils import _to_modin
@@ -21,7 +20,6 @@ def _resolve_datasource_parameters(bulk_read: bool, *args: Any, **kwargs: Any) -
     if bulk_read:
         return {
             "datasource": ArrowParquetBaseDatasource(*args, **kwargs),
-            "meta_provider": FastFileMetadataProvider(),
         }
     else:
         kwargs.pop("path_root")

awswrangler/distributed/ray/modin/s3/_read_text.py

@@ -8,7 +8,6 @@
 import modin.pandas as pd
 from pyarrow import csv
 from ray.data import read_datasource
-from ray.data.datasource.file_meta_provider import FastFileMetadataProvider
 
 from awswrangler import exceptions
 from awswrangler.distributed.ray.datasources import (
@@ -169,7 +168,6 @@ def _read_text_distributed(
             version_ids=version_ids,
             s3_additional_kwargs=s3_additional_kwargs,
             pandas_kwargs=pandas_kwargs,
-            meta_provider=FastFileMetadataProvider(),
             **configuration,
         ),
         override_num_blocks=override_num_blocks,