Skip to content

Refactor: remove unnecessary async for ArrowReader::read #1608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 19, 2025
Merged

Refactor: remove unnecessary async for ArrowReader::read #1608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a25371b
Remove unnecessary async for ArrowReader::read
ZENOTME Aug 16, 2025
939dd78
Merge branch 'main' into remove_unnecessary_async
liurenjie1024 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions crates/iceberg/src/arrow/reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ pub struct ArrowReader {
impl ArrowReader {
/// Take a stream of FileScanTasks and reads all the files.
/// Returns a stream of Arrow RecordBatches containing the data from the files
pub async fn read(self, tasks: FileScanTaskStream) -> Result<ArrowRecordBatchStream> {
pub fn read(self, tasks: FileScanTaskStream) -> Result<ArrowRecordBatchStream> {
let file_io = self.file_io.clone();
let batch_size = self.batch_size;
let concurrency_limit_data_files = self.concurrency_limit_data_files;
Expand Down Expand Up @@ -1751,7 +1751,6 @@ message schema {

let result = reader
.read(tasks)
.await
.unwrap()
.try_collect::<Vec<RecordBatch>>()
.await
Expand Down
7 changes: 1 addition & 6 deletions crates/iceberg/src/scan/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,7 @@ impl TableScan {
arrow_reader_builder = arrow_reader_builder.with_batch_size(batch_size);
}

arrow_reader_builder
.build()
.read(self.plan_files().await?)
.await
arrow_reader_builder.build().read(self.plan_files().await?)
}

/// Returns a reference to the column names of the table scan.
Expand Down Expand Up @@ -1332,14 +1329,12 @@ pub mod tests {
let batch_stream = reader
.clone()
.read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
.await
.unwrap();
let batch_1: Vec<_> = batch_stream.try_collect().await.unwrap();

let reader = ArrowReaderBuilder::new(fixture.table.file_io().clone()).build();
let batch_stream = reader
.read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
.await
.unwrap();
let batch_2: Vec<_> = batch_stream.try_collect().await.unwrap();

Expand Down
Loading